diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f65c83b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +.git +.github +.vscode +.idea + +__pycache__ +*.pyc +*.pyo +*.log + +.venv +venv +env + +deploy/deps +deploy/tmp +deploy/recordings + +*.pth +*.pt +*.safetensors +*.onnx \ No newline at end of file diff --git a/.github/workflows/build-runpod-h200.yml b/.github/workflows/build-runpod-h200.yml new file mode 100644 index 0000000..faf94fd --- /dev/null +++ b/.github/workflows/build-runpod-h200.yml @@ -0,0 +1,76 @@ +name: Build RunPod H200 container image + +on: + workflow_dispatch: + push: + branches: + - main + - "agent/**" + paths: + - .dockerignore + - .github/workflows/build-runpod-h200.yml + - Dockerfile.h200 + - deploy/** + - runpod/** + +permissions: + contents: read + packages: write + +concurrency: + group: runpod-h200-image-${{ github.ref }} + cancel-in-progress: true + +env: + REGISTRY: ghcr.io + IMAGE_NAME: samuellucky2424-afk/joyai-video-edit + +jobs: + build-and-push: + name: Build and publish H200 image + runs-on: ubuntu-24.04 + timeout-minutes: 330 + + steps: + - name: Free disk space for the CUDA image + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: false + + - name: Check out the repository + uses: actions/checkout@v6 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish the H200 container image + id: build + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: ./Dockerfile.h200 + platforms: linux/amd64 + push: true + provenance: false + build-args: | + MAX_JOBS=2 + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:runpod-h200 + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:h200-sha-${{ github.sha }} + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.description=JoyAI Video Edit for RunPod H200 + + - name: Show immutable deployment reference + run: echo "${REGISTRY}/${IMAGE_NAME}@${{ steps.build.outputs.digest }}" diff --git a/.github/workflows/build-runpod-image.yml b/.github/workflows/build-runpod-image.yml new file mode 100644 index 0000000..f2ab5b1 --- /dev/null +++ b/.github/workflows/build-runpod-image.yml @@ -0,0 +1,65 @@ +name: Build RunPod container image + +on: + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + group: runpod-image-${{ github.ref }} + cancel-in-progress: false + +env: + REGISTRY: ghcr.io + IMAGE_NAME: samuellucky2424-afk/joyai-video-edit + +jobs: + build-and-push: + name: Build and publish RTX PRO 6000 image + runs-on: ubuntu-24.04 + timeout-minutes: 330 + + steps: + - name: Free disk space for the CUDA image + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: false + + - name: Check out the RunPod branch + uses: actions/checkout@v6 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and publish the container image + id: build + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64 + push: true + build-args: | + MAX_JOBS=2 + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:runpod-rtx-pro-6000 + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }} + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.description=JoyAI Video Edit for RunPod RTX PRO 6000 + + - name: Show immutable deployment reference + run: echo "${REGISTRY}/${IMAGE_NAME}@${{ steps.build.outputs.digest }}" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..14620fe --- /dev/null +++ b/Dockerfile @@ -0,0 +1,80 @@ +# syntax=docker/dockerfile:1.7 + +FROM nvidia/cuda:12.8.1-devel-ubuntu22.04 + +ARG DEBIAN_FRONTEND=noninteractive +ARG MAX_JOBS=8 + +ENV PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + CUDA_HOME=/usr/local/cuda \ + TORCH_CUDA_ARCH_LIST=12.0 \ + JOYOMNI_OPS_CUDA_ARCHS=120a \ + MAX_JOBS=${MAX_JOBS} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + ffmpeg \ + git \ + libglib2.0-0 \ + libgl1 \ + ninja-build \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + python-is-python3 \ + && rm -rf /var/lib/apt/lists/* \ + && python3 -m pip install --upgrade pip setuptools wheel + +WORKDIR /opt/joyai + +COPY deploy/requirements.txt /tmp/requirements.txt + +RUN python3 -m pip install -r /tmp/requirements.txt + +# Install patched SageAttention for RTX PRO 6000 Blackwell. +COPY deploy/sageattention-cudagraph-stream.patch /tmp/sageattention.patch + +RUN git clone https://github.com/thu-ml/SageAttention.git /tmp/SageAttention \ + && git -C /tmp/SageAttention checkout d1a57a546c3d395b1ffcbeecc66d81db76f3b4b5 \ + && git -C /tmp/SageAttention apply /tmp/sageattention.patch \ + && cd /tmp/SageAttention \ + && EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" python3 setup.py install \ + && rm -rf /tmp/SageAttention + +# Build the repository's original FP8 CUDA operations. +COPY deploy/joyomni_ops /opt/joyai/deploy/joyomni_ops + +RUN git clone https://github.com/NVIDIA/cutlass.git /tmp/cutlass \ + && git -C /tmp/cutlass checkout dcf215af \ + && JOYOMNI_OPS_CUTLASS_DIR=/tmp/cutlass \ + python3 -m pip install --no-build-isolation /opt/joyai/deploy/joyomni_ops \ + && rm -rf /tmp/cutlass /root/.cache/pip + +COPY . /opt/joyai + +RUN chmod +x /opt/joyai/deploy/run_server.sh \ + && mkdir -p /runpod-volume/joyai \ + && mkdir -p /tmp/joyomni-recordings + +ENV JOYOMNI_DEVICE=cuda:0 \ + JOYOMNI_HOST=0.0.0.0 \ + JOYOMNI_PORT=8080 \ + JOYOMNI_CKPT_ROOT=/runpod-volume/joyai/checkpoints \ + JOYOMNI_WIDTH=840 \ + JOYOMNI_HEIGHT=480 \ + JOYOMNI_FPS=24 \ + JOYOMNI_FP8_IMG=1 \ + JOYOMNI_FP8_TXT=1 \ + JOYOMNI_CUDA_GRAPH=1 \ + JOYOMNI_SAGE_ATTN=1 \ + JOYOMNI_TXT_PARALLEL=1 \ + JOYOMNI_RECORD_DIR=/tmp/joyomni-recordings + +EXPOSE 8080 8081 + +CMD ["python3", "/opt/joyai/runpod/start.py"] \ No newline at end of file diff --git a/Dockerfile.h200 b/Dockerfile.h200 new file mode 100644 index 0000000..81b09f3 --- /dev/null +++ b/Dockerfile.h200 @@ -0,0 +1,119 @@ +# syntax=docker/dockerfile:1.7 + +ARG PYTORCH_IMAGE_VERSION=2.9.1-cuda12.8-cudnn9 + +# Compile the Hopper-specific CUDA extensions in a development image. Nothing +# from this stage is shipped except the two finished Python wheels. +FROM pytorch/pytorch:${PYTORCH_IMAGE_VERSION}-devel AS extensions + +ARG DEBIAN_FRONTEND=noninteractive +ARG MAX_JOBS=8 + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + CUDA_HOME=/usr/local/cuda \ + TORCH_CUDA_ARCH_LIST=9.0 \ + JOYOMNI_OPS_CUDA_ARCHS=90a \ + MAX_JOBS=${MAX_JOBS} + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + git \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* \ + && python -m pip install --upgrade pip setuptools wheel + +WORKDIR /opt/joyai +RUN mkdir -p /wheels + +# SageAttention supports Hopper GPUs. Compile its CUDA extension for H200 +# (sm_90) and keep only the resulting wheel. +COPY deploy/sageattention-cudagraph-stream.patch /tmp/sageattention.patch + +RUN git clone https://github.com/thu-ml/SageAttention.git /tmp/SageAttention \ + && git -C /tmp/SageAttention checkout d1a57a546c3d395b1ffcbeecc66d81db76f3b4b5 \ + && git -C /tmp/SageAttention apply /tmp/sageattention.patch \ + && cd /tmp/SageAttention \ + && EXT_PARALLEL=4 NVCC_APPEND_FLAGS="--threads 8" \ + python setup.py bdist_wheel --dist-dir /wheels \ + && rm -rf /tmp/SageAttention + +# Build JoyAI's FP8 CUDA operations for Hopper (sm_90a) as a wheel. +# CUTLASS WGMMA instructions are architecture-accelerated and abort at runtime +# when the extension is compiled only for the generic sm_90 target. +COPY deploy/joyomni_ops /opt/joyai/deploy/joyomni_ops + +RUN git clone https://github.com/NVIDIA/cutlass.git /tmp/cutlass \ + && git -C /tmp/cutlass checkout dcf215af \ + && JOYOMNI_OPS_CUTLASS_DIR=/tmp/cutlass \ + python -m pip wheel --no-build-isolation --no-deps \ + --wheel-dir /wheels /opt/joyai/deploy/joyomni_ops \ + && rm -rf /tmp/cutlass /root/.cache/pip + + +# The final image already contains CUDA 12.8, cuDNN 9, PyTorch 2.9.1, and +# torchvision. Keep the small host compiler toolchain as an escape hatch for +# deployments that explicitly re-enable Torch Inductor VAE compilation. +# CUDA extensions and the full CUDA development toolchain stay in the builder. +FROM pytorch/pytorch:${PYTORCH_IMAGE_VERSION}-runtime + +ARG DEBIAN_FRONTEND=noninteractive + +ENV PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + CC=gcc \ + CXX=g++ + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + ffmpeg \ + libglib2.0-0 \ + libgl1 \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/joyai + +COPY deploy/requirements-h200-runtime.txt /tmp/requirements.txt +RUN python -m pip install -r /tmp/requirements.txt \ + && rm -f /tmp/requirements.txt + +COPY --from=extensions /wheels /tmp/wheels +RUN python -m pip install --no-deps /tmp/wheels/*.whl \ + && rm -rf /tmp/wheels + +COPY . /opt/joyai + +RUN chmod +x /opt/joyai/deploy/run_server.sh \ + && mkdir -p /runpod-volume/joyai + +ENV JOYOMNI_DEVICE=cuda:0 \ + JOYOMNI_HOST=0.0.0.0 \ + JOYOMNI_PORT=8080 \ + JOYOMNI_CKPT_ROOT=/runpod-volume/joyai/checkpoints \ + JOYOMNI_CACHE_ROOT=/runpod-volume/joyai/cache/h200-torch291-cu128 \ + JOYOMNI_CACHE_READY_MARKER=/runpod-volume/joyai/cache/h200-torch291-cu128/ready.json \ + JOYOMNI_PRELOAD=1 \ + JOYOMNI_WIDTH=840 \ + JOYOMNI_HEIGHT=480 \ + JOYOMNI_FPS=20 \ + JOYOMNI_FP8_IMG=1 \ + JOYOMNI_FP8_TXT=1 \ + JOYOMNI_CUDA_GRAPH=1 \ + JOYOMNI_SAGE_ATTN=1 \ + JOYOMNI_TXT_PARALLEL=1 \ + JOYOMNI_VAE_COMPILE=1 \ + JOYOMNI_VAE_COMPILE_STRICT=1 \ + JOYOMNI_LOAD_WARMUP_STRICT=1 \ + JOYOMNI_FULL_WARMUP_TIMEOUT_SECONDS=300 \ + JOYOMNI_WARMUP_BOTH_ORIENTATIONS=0 \ + JOYOMNI_WARMUP_REFERENCE_BUCKETS=0 \ + JOYOMNI_RECORD_ENABLED=0 \ + JOYOMNI_ONLINE_GATE_ENABLED=0 + +EXPOSE 8080 8081 + +CMD ["python", "/opt/joyai/runpod/start.py"] diff --git a/deploy/requirements-h200-runtime.txt b/deploy/requirements-h200-runtime.txt new file mode 100644 index 0000000..50955f6 --- /dev/null +++ b/deploy/requirements-h200-runtime.txt @@ -0,0 +1,16 @@ +transformers>=4.57.1,<4.58 +accelerate==1.10.1 +diffusers==0.36.0 +einops==0.8.2 +numpy==2.2.6 +pillow==12.2.0 +opencv-python-headless==4.13.0.92 +av==13.1.0 +imageio-ffmpeg==0.6.0 +fastapi==0.117.1 +uvicorn==0.37.0 +uvloop==0.22.1 +ninja==1.13.0 +websockets==16.0 +openai==2.41.0 +loguru==0.7.3 diff --git a/deploy/run_server.sh b/deploy/run_server.sh index bb47f8f..9c26c96 100755 --- a/deploy/run_server.sh +++ b/deploy/run_server.sh @@ -26,13 +26,17 @@ fi cd "$HERE" -export TORCHINDUCTOR_CACHE_DIR="$HERE/deps/cache/torchinductor" -export TRITON_CACHE_DIR="$HERE/deps/cache/triton" -export CUDA_CACHE_PATH="$HERE/deps/cache/nv_compute" -export TORCHINDUCTOR_FX_GRAPH_CACHE=1 +CACHE_ROOT="${JOYOMNI_CACHE_ROOT:-$HERE/deps/cache}" +export TORCHINDUCTOR_CACHE_DIR="${TORCHINDUCTOR_CACHE_DIR:-$CACHE_ROOT/torchinductor}" +export TRITON_CACHE_DIR="${TRITON_CACHE_DIR:-$CACHE_ROOT/triton}" +export CUDA_CACHE_PATH="${CUDA_CACHE_PATH:-$CACHE_ROOT/nv_compute}" +export TORCHINDUCTOR_FX_GRAPH_CACHE="${TORCHINDUCTOR_FX_GRAPH_CACHE:-1}" export PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}" mkdir -p "$TORCHINDUCTOR_CACHE_DIR" "$TRITON_CACHE_DIR" "$CUDA_CACHE_PATH" +echo "JoyAI compile cache root: $CACHE_ROOT" +echo "JoyAI VAE compile: ${JOYOMNI_VAE_COMPILE:-1} (strict=${JOYOMNI_VAE_COMPILE_STRICT:-0})" + export PYTHONUNBUFFERED=1 export PYTHONPATH="$HERE" @@ -43,6 +47,33 @@ export JOYOMNI_SAGE_ATTN="${JOYOMNI_SAGE_ATTN:-1}" export JOYOMNI_TXT_PARALLEL="${JOYOMNI_TXT_PARALLEL:-1}" RECORD_DIR="${JOYOMNI_RECORD_DIR:-$HERE/recordings}" +RECORD_ENABLED="${JOYOMNI_RECORD_ENABLED:-1}" +ONLINE_GATE_ENABLED="${JOYOMNI_ONLINE_GATE_ENABLED:-1}" +EXTRA_ARGS=() + +case "${RECORD_ENABLED,,}" in + 1|true|yes|on) + EXTRA_ARGS+=(--record-dir "$RECORD_DIR") + ;; + 0|false|no|off) + ;; + *) + echo "JOYOMNI_RECORD_ENABLED must be one of: 1, 0, true, false, yes, no, on, off" >&2 + exit 2 + ;; +esac + +case "${ONLINE_GATE_ENABLED,,}" in + 1|true|yes|on) + ;; + 0|false|no|off) + EXTRA_ARGS+=(--no-online-gate) + ;; + *) + echo "JOYOMNI_ONLINE_GATE_ENABLED must be one of: 1, 0, true, false, yes, no, on, off" >&2 + exit 2 + ;; +esac CKPT_ROOT="${JOYOMNI_CKPT_ROOT:-$HERE/deps/checkpoints}" DIT_CKPT="${JOYOMNI_DIT_CKPT:-$CKPT_ROOT/JoyAI-Video-Edit/dit/joyai_video_edit_dit_0811.pth}" @@ -61,7 +92,6 @@ python xvideo/serving/serve_joyomni_streaming.py \ --text-encoder-ckpt "$TE_CKPT" \ --face-detector-onnx "$FACE_ONNX" \ --person-detector-onnx "$PERSON_ONNX" \ - --record-dir "$RECORD_DIR" \ --device "$DEVICE" \ --vae-encode-device "$DEVICE" \ --vae-decode-device "$DEVICE" \ @@ -70,4 +100,5 @@ python xvideo/serving/serve_joyomni_streaming.py \ --width "${JOYOMNI_WIDTH:-840}" --height "${JOYOMNI_HEIGHT:-480}" \ --fps "${JOYOMNI_FPS:-24}" \ --host "$HOST" --port "$PORT" \ + "${EXTRA_ARGS[@]}" \ "$@" diff --git a/deploy/static/index.html b/deploy/static/index.html index b7d5ff2..75f3391 100644 --- a/deploy/static/index.html +++ b/deploy/static/index.html @@ -162,8 +162,6 @@ -
-