Merge branch 'main' into grug_moe_heuristic

Author: ClassicLarry

.agents/skills/fix-docs/SKILL.md

@@ -1,3 +1,8 @@
+---
+name: fix-docs
+description: Fix markdown docs in `lib/iris`, `lib/zephyr`, and `lib/fray` to align with Marin's agent-doc principles. Use when asked to repair, modernize, or de-rot docs in those directories.
+---
+
 Your task is to fix the markdown docs within `lib/iris`, `lib/zephyr` and `lib/fray` so that they maximally comply with the principles below. Do NOT fix docs outside of the aforementioned directories.
 
 Your output: You will dispatch sub-agents that will (1) thoroughly parse the code and the docs and (2) make all the documentation changes that are deemed appropriate, locally. You will commit the changes locally into a single commit, inform the user of the commit, and summarize the changes you made. Under no circumstances should you push any commit to the repo without explicit approval from the user.

.github/workflows/iris-cloud-smoke-gcp.yaml

@@ -171,10 +171,49 @@ jobs:
         if: failure()
         env:
           LOG_DIR: ${{ github.workspace }}/iris-cloud-logs
+          LABEL_PREFIX: ${{ steps.label.outputs.prefix }}
+          PROJECT: ${{ secrets.GCP_PROJECT_ID }}
         run: |
           mkdir -p "$LOG_DIR"
+          MANAGED_LABEL="iris-${LABEL_PREFIX}-managed"
+          CONTROLLER_LABEL="iris-${LABEL_PREFIX}-controller"
+
+          # Always try to grab docker/host logs directly from every VM we
+          # spun up. This works even when the controller never became
+          # reachable (e.g. tunnel never opened, controller crashed during
+          # boot), which is exactly the case where these logs matter most.
+          gcloud compute instances list \
+            --project="$PROJECT" \
+            --filter="labels.${MANAGED_LABEL}=true OR labels.${CONTROLLER_LABEL}=true" \
+            --format="csv[no-heading](name,zone,labels.list())" 2>/dev/null \
+          | while IFS=, read -r name zone labels; do
+              [ -z "$name" ] && continue
+              role=worker
+              case "$labels" in *"$CONTROLLER_LABEL"*) role=controller ;; esac
+              echo "Fetching host logs from $role $name ($zone)"
+              out="$LOG_DIR/${role}-${name}"
+              gcloud compute ssh "$name" \
+                --project="$PROJECT" --zone="$zone" \
+                --impersonate-service-account="$IRIS_CONTROLLER_SERVICE_ACCOUNT" \
+                --ssh-key-file ~/.ssh/google_compute_engine \
+                --quiet \
+                --command '
+                set +e
+                echo "=== docker ps -a ==="
+                sudo docker ps -a
+                for cid in $(sudo docker ps -aq); do
+                  echo "=== docker logs $cid ==="
+                  sudo docker logs --timestamps --tail 5000 "$cid" 2>&1
+                done
+                echo "=== startup script ==="
+                sudo journalctl -u google-startup-scripts.service --no-pager 2>&1 | tail -n 2000
+                echo "=== kernel/cloud-init ==="
+                sudo journalctl -u cloud-final.service --no-pager 2>&1 | tail -n 500
+              ' > "${out}.log" 2>&1 || echo "ssh to $name failed (see ${out}.log)"
+            done
+
           if [ -z "$IRIS_CONTROLLER_URL" ]; then
-            echo "No controller URL, skipping log collection"
+            echo "No controller URL, skipping RPC-based log collection"
             exit 0
           fi
           cd lib/iris

.github/workflows/iris-coreweave-ci.yaml

@@ -104,7 +104,7 @@ jobs:
         run: |
           cd lib/iris && uv run --group dev iris -v \
             --config=examples/coreweave-ci.yaml \
-            cluster start
+            cluster start --fresh
 
       - name: Run integration tests
         env:
@@ -185,9 +185,14 @@ jobs:
 
       - name: Capture failure diagnostics
         if: failure()
+        env:
+          LOG_DIR: ${{ github.workspace }}/iris-cw-logs
         run: |
           export KUBECONFIG=~/.kube/coreweave-iris
-          echo "=== Controller logs ==="
+          mkdir -p "$LOG_DIR"
+
+          # Stream to the GH Actions log for quick triage…
+          echo "=== Controller logs (tail) ==="
           kubectl -n "$IRIS_NAMESPACE" logs -l app=iris-controller --tail=500 || true
           echo "=== Controller pod describe ==="
           kubectl -n "$IRIS_NAMESPACE" describe pod -l app=iris-controller || true
@@ -196,6 +201,36 @@ jobs:
           echo "=== Warning events ==="
           kubectl -n "$IRIS_NAMESPACE" get events --sort-by='.lastTimestamp' --field-selector type!=Normal || true
 
+          # …and also persist per-pod logs + describe so failures in worker
+          # containers are recoverable from the uploaded artifact, not just
+          # the controller's view.
+          kubectl -n "$IRIS_NAMESPACE" logs -l app=iris-controller --tail=-1 --all-containers \
+            > "$LOG_DIR/controller.log" 2>&1 || true
+          kubectl -n "$IRIS_NAMESPACE" logs -l app=iris-controller --tail=-1 --all-containers --previous \
+            > "$LOG_DIR/controller-previous.log" 2>&1 || true
+          kubectl -n "$IRIS_NAMESPACE" describe pod -l app=iris-controller \
+            > "$LOG_DIR/controller-describe.txt" 2>&1 || true
+
+          for pod in $(kubectl -n "$IRIS_NAMESPACE" get pods -l "$IRIS_MANAGED_LABEL=true" -o name 2>/dev/null); do
+            safe=$(echo "$pod" | tr '/' '-')
+            kubectl -n "$IRIS_NAMESPACE" logs "$pod" --tail=-1 --all-containers \
+              > "$LOG_DIR/${safe}.log" 2>&1 || true
+            kubectl -n "$IRIS_NAMESPACE" describe "$pod" \
+              > "$LOG_DIR/${safe}-describe.txt" 2>&1 || true
+          done
+
+          kubectl -n "$IRIS_NAMESPACE" get events --sort-by='.lastTimestamp' \
+            > "$LOG_DIR/events.txt" 2>&1 || true
+
+      - name: Upload failure diagnostics
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: iris-cw-ci-logs
+          path: iris-cw-logs/
+          retention-days: 14
+          if-no-files-found: ignore
+
       - name: Set commit status to result
         if: always() && github.event_name == 'issue_comment'
         env:

.github/workflows/iris-dev-restart.yaml

@@ -2,8 +2,8 @@ name: Iris - Dev Cluster Daily Restart
 
 on:
   schedule:
-    # Daily at 06:00 UTC
-    - cron: "0 6 * * *"
+    # Daily at 05:00 UTC — staggered before canary ferry (06:00 UTC)
+    - cron: "0 5 * * *"
   workflow_dispatch:
 
 permissions:

.github/workflows/marin-canary-ferry-cw.yaml

@@ -58,7 +58,7 @@ jobs:
           enable-cache: true
 
       - name: Install dependencies
-        run: uv sync --all-packages --extra=cpu --no-default-groups
+        run: uv sync --all-packages --extra=cpu --extra=controller --no-default-groups
 
       - name: Write CoreWeave kubeconfig
         run: |
@@ -195,7 +195,7 @@ jobs:
             Read .agents/skills/canary-triage/SKILL.md and follow it.
           claude_args: |
             --model opus
-            --max-turns 50
+            --max-turns 500
             --allowedTools "Bash(kubectl:*),Bash(gh:*),Bash(.venv/bin/iris:*),Bash(.venv/bin/python:*),Bash(cat:*),Bash(jq:*),Bash(head:*),Bash(tail:*),Bash(grep:*)"
         env:
           CANARY_LANE: gpu

.github/workflows/marin-canary-ferry.yaml

@@ -165,7 +165,7 @@ jobs:
             Read .agents/skills/canary-triage/SKILL.md and follow it.
           claude_args: |
             --model opus
-            --max-turns 50
+            --max-turns 500
             --allowedTools "Bash(gh:*),Bash(.venv/bin/iris:*),Bash(.venv/bin/python:*),Bash(cat:*),Bash(jq:*),Bash(head:*),Bash(tail:*),Bash(grep:*)"
         env:
           CANARY_LANE: tpu

.github/workflows/marin-datakit-smoke.yaml

@@ -19,7 +19,7 @@ jobs:
       cancel-in-progress: true
     env:
       SMOKE_RUN_ID: datakit-smoke-${{ github.run_id }}-${{ github.run_attempt }}
-      # MARIN_PREFIX is defaulted by the ferry entrypoint to marin_temp_bucket(ttl_days=1).
+      FERRY_STATUS_PATH: gs://marin-tmp-us-central1/ttl=1d/ci/datakit-smoke-${{ github.run_id }}-${{ github.run_attempt }}/ferry_run_status.json
       WANDB_ENTITY: marin-community
       WANDB_PROJECT: marin
       IRIS_CONFIG: lib/iris/examples/marin-dev.yaml
@@ -70,6 +70,7 @@ jobs:
             job run --no-wait \
             --memory=2G --disk=4G --cpu=1 --extra=cpu \
             -e SMOKE_RUN_ID "$SMOKE_RUN_ID" \
+            -e FERRY_STATUS_PATH "$FERRY_STATUS_PATH" \
             -e WANDB_ENTITY "$WANDB_ENTITY" \
             -e WANDB_PROJECT "$WANDB_PROJECT" \
             -e WANDB_API_KEY "$WANDB_API_KEY" \
@@ -113,12 +114,24 @@ jobs:
             esac
           done
 
+      - name: Read ferry status
+        id: ferry_status
+        shell: bash -l {0}
+        run: |
+          PREFIX=$(.venv/bin/python -c "
+          import json
+          from rigging.filesystem import url_to_fs
+          fs, _ = url_to_fs('$FERRY_STATUS_PATH')
+          with fs.open('$FERRY_STATUS_PATH') as f:
+              print(json.load(f)['marin_prefix'])
+          ")
+          echo "marin_prefix=$PREFIX" >> "$GITHUB_OUTPUT"
+          echo "Ferry output prefix: $PREFIX"
+
       - name: Validate datakit smoke outputs
         shell: bash -l {0}
         env:
-          SMOKE_RUN_ID: ${{ env.SMOKE_RUN_ID }}
-          # MARIN_PREFIX intentionally unset — validate script defaults via marin_temp_bucket,
-          # matching the ferry entrypoint default.
+          MARIN_PREFIX: ${{ steps.ferry_status.outputs.marin_prefix }}
         run: .venv/bin/python scripts/datakit/validate_ferry_outputs.py
 
       - name: Capture failure diagnostics
@@ -143,7 +156,7 @@ jobs:
             Read .agents/skills/canary-triage/SKILL.md and follow it.
           claude_args: |
             --model opus
-            --max-turns 50
+            --max-turns 500
             --allowedTools "Bash(gh:*),Bash(.venv/bin/iris:*),Bash(.venv/bin/python:*),Bash(cat:*),Bash(jq:*),Bash(head:*),Bash(tail:*),Bash(grep:*)"
         env:
           CANARY_LANE: datakit-smoke

.github/workflows/marin-infra-dashboard.yaml

@@ -0,0 +1,60 @@
+name: Marin - Infra Dashboard
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      should_run: ${{ steps.filter.outputs.relevant }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            relevant:
+              - 'infra/status-page/**'
+              - '.github/workflows/marin-infra-dashboard.yaml'
+
+  build:
+    needs: changes
+    if: needs.changes.outputs.should_run == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: infra/status-page
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: infra/status-page/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Typecheck
+        run: npm run typecheck
+
+      - name: Build
+        run: npm run build

experiments/defaults.py

@@ -206,6 +206,9 @@ def default_tokenize(
     *,
     sample_count: int | VersionedValue[int] | None = None,
     is_validation: bool = False,
+    levanter_batch_size: int | None = None,
+    resources: ResourceConfig | None = None,
+    worker_resources: ResourceConfig | None = None,
 ) -> ExecutorStep:
     """
     Tokenizes a dataset using the specified tokenizer and Levanter's tokenization infrastructure.
@@ -228,6 +231,11 @@ def default_tokenize(
         An ExecutorStep that represents the tokenized dataset.
     """
 
+    # Common kwargs for config constructors
+    extra_kwargs: dict = {}
+    if worker_resources is not None:
+        extra_kwargs["worker_resources"] = worker_resources
+
     # sniff out if it's a HuggingFace dataset
     if isinstance(dataset, HfDatasetSpec):
         config = HfTokenizeConfig(
@@ -237,6 +245,8 @@ def default_tokenize(
             tokenizer=ensure_versioned(tokenizer),
             format=format,
             sample_count=ensure_versioned(sample_count) if sample_count is not None else None,
+            levanter_batch_size=levanter_batch_size,
+            **extra_kwargs,
         )
     elif (
         isinstance(dataset, str)
@@ -250,6 +260,8 @@ def default_tokenize(
             tokenizer=ensure_versioned(tokenizer),
             format=format,
             sample_count=ensure_versioned(sample_count) if sample_count is not None else None,
+            levanter_batch_size=levanter_batch_size,
+            **extra_kwargs,
         )
     else:
         config = TokenizeConfig(
@@ -259,14 +271,16 @@ def default_tokenize(
             tokenizer=ensure_versioned(tokenizer),
             format=format,
             sample_count=ensure_versioned(sample_count) if sample_count is not None else None,
+            levanter_batch_size=levanter_batch_size,
+            **extra_kwargs,
         )
 
     return ExecutorStep(
         name=os.path.join("tokenized", name),
         description=f"Tokenize raw text using the {tokenizer} tokenizer.",
         fn=remote(
             tokenize,
-            resources=ResourceConfig.with_cpu(cpu=4, ram="16g", disk="10g"),
+            resources=resources or ResourceConfig.with_cpu(cpu=4, ram="16g", disk="10g"),
             pip_dependency_groups=["cpu"],
             env_vars={
                 "TRANSFORMERS_NO_TORCH": "1",

experiments/ferries/datakit_ferry.py

@@ -7,10 +7,11 @@
 Output paths are placed under ``$MARIN_PREFIX/datakit-smoke/$SMOKE_RUN_ID/...``.
 """
 
+import json
 import logging
 import os
 
-from rigging.filesystem import marin_temp_bucket
+from rigging.filesystem import marin_temp_bucket, url_to_fs
 from rigging.log_setup import configure_logging
 
 from fray import ResourceConfig
@@ -109,14 +110,30 @@ def build_steps(run_id: str) -> list[StepSpec]:
     return [downloaded, normalized, deduped, consolidated, tokenized]
 
 
+def _write_status(status: str, marin_prefix: str) -> None:
+    """Write ferry run status to FERRY_STATUS_PATH if set."""
+    status_path = os.environ.get("FERRY_STATUS_PATH")
+    if not status_path:
+        return
+    payload = json.dumps({"status": status, "marin_prefix": marin_prefix})
+    fs, _ = url_to_fs(status_path)
+    with fs.open(status_path, "w") as f:
+        f.write(payload)
+    logger.info("Wrote ferry status to %s", status_path)
+
+
 def main() -> None:
     configure_logging()
     if not os.environ.get("MARIN_PREFIX"):
         os.environ["MARIN_PREFIX"] = marin_temp_bucket(ttl_days=1)
 
-    logger.info("MARIN_PREFIX defaulted to %s", os.environ["MARIN_PREFIX"])
+    marin_prefix = os.environ["MARIN_PREFIX"]
+    logger.info("MARIN_PREFIX defaulted to %s", marin_prefix)
     run_id = os.environ["SMOKE_RUN_ID"]
+
+    _write_status("running", marin_prefix)
     StepRunner().run(build_steps(run_id))
+    _write_status("succeeded", marin_prefix)
 
 
 if __name__ == "__main__":