first version of doc

Author: qchapp

.github/workflows/docs.yml

@@ -0,0 +1,67 @@
+name: Deploy Sphinx docs to GitHub Pages
+
+on:
+  push:
+    branches:
+      - main
+      - feature/doc
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Only one deployment runs at a time; ongoing deployments are not cancelled.
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    name: Build documentation
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+          cache-dependency-path: docs/requirements.txt
+
+      - name: Install Sphinx dependencies
+        run: pip install -r docs/requirements.txt
+
+      # Install the package itself in no-deps, no-build-isolation mode so that
+      # autodoc can import mmirage.  Heavy runtime dependencies (torch, sglang,
+      # etc.) are mocked in conf.py and therefore do not need to be installed.
+      - name: Install mmirage (no heavy deps)
+        run: pip install --no-deps -e .
+
+      - name: Build HTML docs
+        run: sphinx-build -b html docs docs/_build/html -W --keep-going
+        env:
+          SPHINXOPTS: "-j auto"
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/_build/html
+
+  deploy:
+    name: Deploy to GitHub Pages
+    needs: build
+    runs-on: ubuntu-latest
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+
+    steps:
+      - name: Deploy
+        id: deployment
+        uses: actions/deploy-pages@v4

docs/.nojekyll

@@ -0,0 +1,8 @@
+# `mmirage.cli` — Command-Line Interface
+
+```{eval-rst}
+.. automodule:: mmirage.cli
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```

docs/_static/.gitkeep

@@ -0,0 +1,34 @@
+# `mmirage.config` — Configuration
+
+## `mmirage.config.config`
+
+Main configuration dataclasses for the MMIRAGE pipeline.
+
+```{eval-rst}
+.. automodule:: mmirage.config.config
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## `mmirage.config.loading`
+
+Dataset loading and sharding configuration.
+
+```{eval-rst}
+.. automodule:: mmirage.config.loading
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## `mmirage.config.utils`
+
+YAML parsing helpers and configuration loaders.
+
+```{eval-rst}
+.. automodule:: mmirage.config.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```

docs/_templates/.gitkeep

@@ -0,0 +1,10 @@
+# Core Modules
+
+```{toctree}
+:maxdepth: 2
+
+loader
+process
+writer
+shard
+```

docs/api/cli.md

@@ -0,0 +1,37 @@
+# `mmirage.core.loader` — Data Loaders
+
+## Base classes
+
+```{eval-rst}
+.. automodule:: mmirage.core.loader.base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## JSONL loader
+
+```{eval-rst}
+.. automodule:: mmirage.core.loader.jsonl
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## HuggingFace local loader
+
+```{eval-rst}
+.. automodule:: mmirage.core.loader.local_hf
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## Loader utilities
+
+```{eval-rst}
+.. automodule:: mmirage.core.loader.utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```

docs/api/config.md

@@ -0,0 +1,48 @@
+# `mmirage.core.process` — Processors
+
+## Variables
+
+```{eval-rst}
+.. automodule:: mmirage.core.process.variables
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## Base processor
+
+```{eval-rst}
+.. automodule:: mmirage.core.process.base
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## Mapper
+
+```{eval-rst}
+.. automodule:: mmirage.core.process.mapper
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## LLM processor
+
+### Configuration
+
+```{eval-rst}
+.. automodule:: mmirage.core.process.processors.llm.config
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+### Implementation
+
+```{eval-rst}
+.. automodule:: mmirage.core.process.processors.llm.llm_processor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```

docs/api/core/index.md

@@ -0,0 +1,67 @@
+# Shard Utilities
+
+## `mmirage.shard_process`
+
+Main entry point for processing a single shard.
+
+```{eval-rst}
+.. automodule:: mmirage.shard_process
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## `mmirage.shard_utils`
+
+Low-level helpers for shard state management, atomic saves, and status markers.
+
+```{eval-rst}
+.. automodule:: mmirage.shard_utils
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## `mmirage.merge_shards`
+
+Dataset shard merging logic.
+
+```{eval-rst}
+.. automodule:: mmirage.merge_shards
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## `mmirage.cli_utils.status`
+
+Shard status checking and retry helpers.
+
+```{eval-rst}
+.. automodule:: mmirage.cli_utils.status
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## `mmirage.cli_utils.slurm`
+
+SLURM submission and job monitoring helpers.
+
+```{eval-rst}
+.. automodule:: mmirage.cli_utils.slurm
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```
+
+## `mmirage.cli_utils.runtime`
+
+Runtime environment setup helpers.
+
+```{eval-rst}
+.. automodule:: mmirage.cli_utils.runtime
+   :members:
+   :undoc-members:
+   :show-inheritance:
+```