feat(gpu): OVHcloud AI Training as a governed engine (server-side)

CHANGELOG.md

@@ -10,6 +10,20 @@
   with `DEBUG=true` is unaffected. See [`SECURITY.md`](SECURITY.md) for the
   disclosed defect and remediation steps. (GLA-22)
 
+## 2.40.1
+
+### Added
+
+- **OVHcloud AI Training as a governed engine** — `openrunner gpu run --provider
+  ovhai --image <img> --gpu-count N "<cmd>"` submits an OVH AI Training managed
+  GPU batch job **server-side** using the org's `ovhcloud` credential
+  (application_key/secret/consumer_key); the keys never touch the client. The API
+  signs the OVH call, submits `/cloud/project/{sid}/ai/job`, and streams
+  status+logs. Data comes from OpenRunner (datasets/artifacts) — OVH
+  object-storage `--volume` mounts are intentionally not used.
+  NOTE: the OVH `/ai/job` request/response shape follows the OVH API but must be
+  smoke-tested against a live OVH project before production use.
+
 ## 2.40.0
 
 ### Changed

sdk/openrunner/__init__.py

@@ -181,7 +181,7 @@
 # openrunner.trace.patch_openai() syntax
 trace.patch_openai = _patch_openai  # type: ignore[attr-defined]
 
-__version__ = "2.40.0"
+__version__ = "2.40.1"
 
 logger = logging.getLogger("openrunner")
 

sdk/openrunner/cli.py

@@ -4244,8 +4244,8 @@ def gpu_run(command, provider, gpu, gpu_count, image, disk_gb, env, ssh_keys, pr
     """
     from openrunner.gpu.runner import run_governed_sync, run_on_gpu_sync
 
-    if provider not in ("forge", "modal") and not gpu:
-        click.echo("Error: --gpu is required (except for forge/modal).", err=True)
+    if provider not in ("forge", "modal", "ovhai") and not gpu:
+        click.echo("Error: --gpu is required (except for forge/modal/ovhai).", err=True)
         raise SystemExit(1)
 
     settings = _load_settings()
@@ -4294,6 +4294,24 @@ def gpu_run(command, provider, gpu, gpu_count, image, disk_gb, env, ssh_keys, pr
     # injected server-side (the token never touches this machine). COMMAND is the
     # App script. A caller who explicitly supplies the token via -e (FORGE_TOKEN /
     # MODAL_TOKEN_SECRET) opts into running the engine client locally instead.
+    # OVH AI Training — governed managed batch job, submitted server-side with the
+    # org's OVH credential (keys never leave the API). COMMAND runs in --image on
+    # --gpu-count GPUs; data comes from OpenRunner (no OVH object-storage volumes).
+    if provider == "ovhai":
+        if image == "pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime":
+            click.echo("Error: --image is required for ovhai (the training image).", err=True)
+            raise SystemExit(1)
+        from openrunner.gpu.runner import run_ovh_training_sync
+        try:
+            result = run_ovh_training_sync(
+                command=command, image=image, gpu=gpu_count, project=project,
+                emit=lambda m: click.echo(m),
+            )
+        except Exception as exc:  # noqa: BLE001
+            click.echo(f"gpu run failed: {exc}", err=True)
+            raise SystemExit(1)
+        raise SystemExit(result["exit_code"])
+
     if provider in ("forge", "modal"):
         _DEFAULT_IMAGE = "pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime"
         local_tok = env_dict.get("FORGE_TOKEN") if provider == "forge" else env_dict.get("MODAL_TOKEN_SECRET")

sdk/openrunner/gpu/api_integration.py

@@ -408,6 +408,41 @@ def get_job(self, job_id: str) -> dict[str, Any] | None:
             logger.warning("get_job failed: %s", e)
             return None
 
+    def ovh_training_run(
+        self, *, image: str, command: str, gpu: int = 1,
+        name: str | None = None, project_id: str | None = None,
+    ) -> dict[str, Any] | None:
+        """POST /api/v1/gpu/ai-training/run -- submit an OVH AI Training job
+        server-side (the OVH credential never leaves the API). Returns
+        {job_id,status} or None."""
+        org = self.org_id()
+        body: dict[str, Any] = {"image": image, "command": command, "gpu": gpu}
+        if name:
+            body["name"] = name
+        if project_id:
+            body["project_id"] = project_id
+        try:
+            r = self._client.post("/gpu/ai-training/run", params={"org_id": org}, json=body)
+            r.raise_for_status()
+            return r.json()
+        except Exception as e:
+            detail = getattr(getattr(e, "response", None), "text", "") or str(e)
+            logger.warning("ovh_training_run failed: %s", detail)
+            return None
+
+    def ovh_training_status(self, job_id: str) -> dict[str, Any] | None:
+        """GET /api/v1/gpu/ai-training/{id} -- poll status + logs."""
+        try:
+            r = self._client.get(
+                f"/gpu/ai-training/{job_id}", params={"org_id": self.org_id()}
+            )
+            if r.status_code >= 400:
+                return None
+            return r.json()
+        except Exception as e:
+            logger.warning("ovh_training_status failed: %s", e)
+            return None
+
     # ------------------------------------------------------------------
     # Lifecycle
     # ------------------------------------------------------------------

sdk/openrunner/gpu/runner.py

@@ -526,6 +526,57 @@ def run_engine_job_sync(
                 "`openrunner runner start` on a box in this org (needs Docker).")
 
 
+def run_ovh_training_sync(
+    *,
+    command: str,
+    image: str,
+    gpu: int = 1,
+    project: str | None = None,
+    name: str | None = None,
+    emit: Callable[[str], None] | None = None,
+    poll_interval: float = 5.0,
+) -> dict[str, Any]:
+    """Governed OVH AI Training run — SERVER-SIDE. Submits a job (the OVH keys
+    stay in the API vault, never on this machine), then polls status + logs until
+    it finishes. COMMAND runs in IMAGE on `gpu` GPUs. Data comes from OpenRunner
+    (datasets/artifacts) — no OVH object-storage volumes. Returns
+    ``{"job_id","exit_code"}``."""
+    from openrunner.gpu import _get_api_client
+
+    say = emit or (lambda m: print(m, flush=True))
+    client = _get_api_client()
+    if client is None:
+        raise RuntimeError("Not logged in. Run 'openrunner login' first.")
+
+    say(f"[openrunner] submitting OVH AI Training job (image={image}, gpu={gpu}) "
+        "— OVH keys stay server-side ...")
+    res = client.ovh_training_run(image=image, command=command, gpu=gpu,
+                                  name=name, project_id=project)
+    if not res or not res.get("job_id"):
+        raise RuntimeError(
+            "failed to submit OVH AI Training job — need an org 'ovhcloud' "
+            "credential (openrunner gpu credentials add --provider ovhcloud ...)."
+        )
+    jid = res["job_id"]
+    say(f"[openrunner] OVH job {jid} ({res.get('status')}) — streaming ...")
+
+    seen = 0
+    while True:
+        time.sleep(poll_interval)
+        st = client.ovh_training_status(jid) or {}
+        logs = st.get("logs") or ""
+        if len(logs) > seen:
+            say(logs[seen:].rstrip("\n"))
+            seen = len(logs)
+        status = st.get("status")
+        if status in ("terminated", "error"):
+            code = st.get("exit_code")
+            if code is None:
+                code = 0 if status == "terminated" else 1
+            say(f"[openrunner] OVH AI Training job {status} (exit={code}).")
+            return {"job_id": jid, "exit_code": int(code)}
+
+
 def run_engine_serverside_sync(
     *,
     provider: str,

sdk/openrunner/install_commands.py

@@ -3599,6 +3599,13 @@ def install_claude_code() -> list[str]:
   SCRIPT (GPU is declared in `@app.function(gpu=...)`, so no `--gpu`), the Run
   auto-terminates when the function returns. Don't hunt for an SSH host/runner.
   For native Forge use, the `forge-deploy` skill covers install/auth/write/run.
+- OVHcloud **AI Training** (managed GPU batch jobs): governed, submitted
+  SERVER-SIDE with the org's OVH credential (the `ovhcloud` app/secret/consumer
+  keys, never on the client). Run:
+  `openrunner gpu run --provider ovhai --image <img> --gpu-count N "<train cmd>"`
+  -- COMMAND runs in the image on N GPUs; poll streams logs; it auto-stops when
+  the command exits. **Do NOT use OVH object-storage `--volume` mounts** -- data
+  comes from OpenRunner (datasets/artifacts). Needs an `ovhcloud` credential.
 - Provider keys are stored encrypted with `openrunner gpu credentials add`
   (e.g. `--provider forge --api-key fg_..`); list/kill pods with
   `openrunner gpu ls` / `openrunner gpu terminate`. Governed compute -- the key

sdk/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openrunner-sdk"
-version = "2.40.0"
+version = "2.40.1"
 description = "OpenRunner SDK - W&B-compatible ML experiment tracking client"
 readme = "README.md"
 license = {text = "MIT"}

sdk/tests/test_gpu/test_ovh_training.py

@@ -0,0 +1,47 @@
+"""SDK OVH AI Training governed run (server-side; OVH keys never on client)."""
+
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from openrunner.gpu import runner as R
+
+
+class _Fake:
+    def __init__(self, statuses):
+        self.run_args = None
+        self._st = list(statuses)
+
+    def ovh_training_run(self, **kwargs):
+        self.run_args = kwargs
+        return {"job_id": "ai-1", "status": "queued"}
+
+    def ovh_training_status(self, job_id):
+        return self._st.pop(0) if self._st else {"status": "terminated", "exit_code": 0}
+
+
+def test_ovh_training_submits_and_streams():
+    fake = _Fake([
+        {"status": "running", "logs": "epoch 1\n"},
+        {"status": "running", "logs": "epoch 1\nepoch 2\n"},
+        {"status": "terminated", "exit_code": 0, "logs": "epoch 1\nepoch 2\ndone\n"},
+    ])
+    lines: list[str] = []
+    with patch("openrunner.gpu._get_api_client", return_value=fake):
+        res = R.run_ovh_training_sync(
+            command="python train.py", image="ovhcom/ai-training-pytorch",
+            gpu=2, emit=lines.append, poll_interval=0,
+        )
+    assert res == {"job_id": "ai-1", "exit_code": 0}
+    assert fake.run_args["image"] == "ovhcom/ai-training-pytorch"
+    assert fake.run_args["gpu"] == 2
+    assert fake.run_args["command"] == "python train.py"
+    assert any("done" in ln for ln in lines)
+
+
+def test_ovh_training_nonzero_exit():
+    fake = _Fake([{"status": "error", "exit_code": 7, "logs": "boom\n"}])
+    with patch("openrunner.gpu._get_api_client", return_value=fake):
+        res = R.run_ovh_training_sync(command="x", image="i", emit=lambda _l: None,
+                                      poll_interval=0)
+    assert res["exit_code"] == 7

src/api/app/api/v1/gpu.py

@@ -96,6 +96,74 @@ async def delete_gpu_credential(
         raise HTTPException(status_code=404, detail="Credential not found")
 
 
+# ---------------------------------------------------------------------------
+# OVHcloud AI Training — governed managed batch jobs. Submitted SERVER-SIDE with
+# the org's OVH credential (stored under provider 'ovhcloud'); the keys never
+# reach the client. No object-storage volumes — data comes from OpenRunner.
+# ---------------------------------------------------------------------------
+
+
+class OvhTrainingRun(BaseModel):
+    image: str = Field(min_length=1, max_length=512)
+    command: str = Field(min_length=1, max_length=16000)
+    gpu: int = Field(default=1, ge=1, le=8)
+    name: str | None = Field(default=None, max_length=60)
+    project_id: UUID | None = None
+
+
+@router.post("/ai-training/run", status_code=201)
+async def ovh_ai_training_run(
+    data: OvhTrainingRun,
+    org_id: UUID = Query(...),
+    _member: OrgMember = Depends(require_org_role()),
+    db: AsyncSession = Depends(get_db),
+):
+    """Submit an OVH AI Training job with the org's OVH credential (server-side).
+    COMMAND runs in IMAGE on `gpu` GPUs; poll `/gpu/ai-training/{id}` for
+    status+logs. Data comes from OpenRunner (datasets/artifacts), not OVH volumes."""
+    import shlex as _shlex
+
+    key = await get_decrypted_key(db, org_id, "ovhcloud")
+    if not key:
+        raise HTTPException(
+            status_code=400,
+            detail="no 'ovhcloud' credential for this org — add it with "
+                   "`openrunner gpu credentials add --provider ovhcloud ...`",
+        )
+    cmd = data.command.strip()
+    command = _shlex.split(cmd[1:-1]) if cmd.startswith("[") else ["bash", "-c", data.command]
+    try:
+        res = await gpu_provision.ovh_training_create(
+            key, image=data.image, gpu=data.gpu, command=command,
+            name=data.name or "openrunner-train",
+        )
+    except gpu_provision.ProvisionError as e:
+        raise HTTPException(status_code=502, detail=str(e))
+    return {"job_id": res["provider_instance_id"], "status": res["status"]}
+
+
+@router.get("/ai-training/{job_id}")
+async def ovh_ai_training_status(
+    job_id: str,
+    org_id: UUID = Query(...),
+    logs: bool = Query(default=True),
+    _member: OrgMember = Depends(require_org_role()),
+    db: AsyncSession = Depends(get_db),
+):
+    """Poll an OVH AI Training job: status/exit_code (+ logs unless logs=0)."""
+    key = await get_decrypted_key(db, org_id, "ovhcloud")
+    if not key:
+        raise HTTPException(status_code=400, detail="no 'ovhcloud' credential for this org")
+    try:
+        st = await gpu_provision.ovh_training_status(key, job_id)
+        out = {"job_id": job_id, **st}
+        if logs:
+            out["logs"] = await gpu_provision.ovh_training_logs(key, job_id)
+        return out
+    except gpu_provision.ProvisionError as e:
+        raise HTTPException(status_code=502, detail=str(e))
+
+
 # ---------------------------------------------------------------------------
 # Server-side engine launch (Forge, Modal). The SERVER runs the engine client
 # via the internal engine-executor sidecar, using the org's decrypted token. The