sp26-optimum-neuron/.github/workflows/doc-build.yml at main · mlberkeley/sp26-optimum-neuron · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
name: Build documentation

on:
  push:
    branches:
      - main
    tags:
      - 'v[0-9]+.[0-9]+.[0-9]+'

    paths:
      - 'docs/source/**'
      - 'docs/assets/**'
      - 'notebooks/**'
      - 'optimum/**'
      - '.github/workflows/doc-build.yml'
  workflow_dispatch:

jobs:
  build_documentation:
    runs-on: ubuntu-22.04
    env:
      COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
      PR_NUMBER: ${{ github.event.number }}
      EVENT_CONTEXT: ${{ toJSON(github.event) }}
      PR_CLONE_URL: ${{ github.event.pull_request.head.repo.clone_url }}

    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: '20'
          cache-dependency-path: "kit/package-lock.json"
      - name: Install Neuronx runtime
        uses: ./.github/actions/install_neuronx_runtime
      - name: Setup virtual environment
        uses: ./.github/actions/setup_venv
      - name: Setup PATH
        run: echo "/home/ubuntu/.local/bin" >> $GITHUB_PATH
      - name: Set environment variables
        run: |
          cd optimum
          version=`echo "$(grep '^__version__ =' neuron/version.py | cut -d '=' -f 2- | xargs)"`

          if [[ $version == *.dev* ]]
          then
            echo "VERSION=main" >> $GITHUB_ENV
          else
            echo "VERSION=v$version" >> $GITHUB_ENV
          fi

          cd ..

      - name: Setup environment
        run: |
          source aws_neuron_venv_pytorch/bin/activate
          uv pip install ".[diffusers]"
          uv pip install "git+https://github.com/huggingface/doc-builder.git"
      - name: Convert notebooks
        shell: bash
        run: |
          mkdir -p docs/source/inference_tutorials/
          mkdir -p docs/source/training_tutorials/
          source aws_neuron_venv_pytorch/bin/activate
          doc-builder notebook-to-mdx notebooks/ec2/text-classification/fine_tune_bert.ipynb --output_dir docs/source/training_tutorials/
          doc-builder notebook-to-mdx notebooks/sagemaker/deploy-llama-3-3-70b.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/sagemaker/deploy-mixtral-8x7b.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/ec2/sentence-transformers/sentence-transformers.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/ec2/text-generation/llama2-13b-chatbot.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/ec2/stable-diffusion/stable-diffusion-txt2img.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/ec2/stable-diffusion/stable-diffusion-xl-txt2img.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/ec2/text-generation/CodeLlama-7B-Compilation.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/ec2/feature-extraction/qwen_embedding.ipynb --output_dir docs/source/inference_tutorials/
          doc-builder notebook-to-mdx notebooks/ie/compare-book-translations.ipynb --output_dir docs/source/inference_tutorials/
          sed 's#](\./#](https://github.com/huggingface/optimum-neuron/blob/main/notebooks/#g' notebooks/README.md > docs/source/notebooks.mdx
      - name: Make documentation
        shell: bash
        run: |
          source aws_neuron_venv_pytorch/bin/activate
          doc-builder build optimum.neuron docs/source/ \
            --repo_name optimum-neuron \
            --build_dir neuron-doc-build/ \
            --version ${{ env.VERSION }} \
            --version_tag_suffix "" \
            --html \
            --clean \
            --notebook_dir docs/notebooks/
          cd  neuron-doc-build/
          mv optimum.neuron optimum-neuron
          doc-builder push optimum-neuron --doc_build_repo_id "hf-doc-build/doc-build" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit $COMMIT_SHA See: https://github.com/huggingface/optimum-neuron/commit/$COMMIT_SHA" --n_retries 5