ci: Don't install things to home directory, add workflow to test release pipelines (#706)

Summary:
Noticed we were getting extra conda packges from the home directory on
the M1 runners. Making this PR to reduce the amount that we're using the
home directory.

Will probably make it unwritable in the future so this is a pre-emptive
step to that.

Also adds a workflow to test the release pipelines without actually releasing a new binary.

Signed-off-by: Eli Uriegas <[email protected]>

Pull Request resolved: #706

Reviewed By: ejguan

Differential Revision: D38406845

Pulled By: seemethere

fbshipit-source-id: 1d312c3e1db34497c546a9b842ac7ba01d2a7ee3

Author: seemethere

.github/workflows/_build_test_upload.yml

@@ -12,6 +12,10 @@ on:
       pytorch_version:
         required: true
         type: string
+      do-upload:
+        required: false
+        default: true
+        type: boolean
     secrets:
       PYTORCH_BINARY_AWS_ACCESS_KEY_ID:
         required: true
@@ -28,24 +32,18 @@ on:
 
 jobs:
   get_release_type:
-    if: inputs.branch != ''
     runs-on: ubuntu-latest
     outputs:
       type: ${{ steps.get_release_type.outputs.type }}
     steps:
       - name: Get Release Type
         run: |
-          if [[ ${{ inputs.branch }} == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then
+          if [[ "${{ inputs.branch }}" == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then
             RELEASE_TYPE=official
-          elif [[ ${{ inputs.branch }} == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
+          elif [[ "${{ inputs.branch }}" == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
             RELEASE_TYPE=test
-          elif [[ ${{ inputs.branch }} == main ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then
-            RELEASE_TYPE=nightly
           else
-            echo "Invalid combination of inputs!"
-            echo "  branch: ${{ inputs.branch }}"
-            echo "  pre_dev_release: ${{ inputs.pre_dev_release }}"
-            exit 1
+            RELEASE_TYPE=nightly
           fi
           echo "Release Type: $RELEASE_TYPE"
           echo "::set-output name=type::$RELEASE_TYPE"
@@ -83,19 +81,23 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Add temp runner environment variables
+        shell: bash -l {0}
+        run: |
+          echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}"
       - name: Install Conda on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
         run: |
-          mkdir -p ~/miniconda3
+          mkdir -p "${MINICONDA_INSTALL_PATH_MACOS}"
           if ${{ startsWith( matrix.os, 'macos-m1' ) }}; then
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           else
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           fi
-          bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
-          rm -rf ~/miniconda3/miniconda.sh
-          echo "$(dirname ~)/$(basename ~)/miniconda3/bin" >> $GITHUB_PATH
+          bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}"
+          rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
+          echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> $GITHUB_PATH
       - name: Setup Python ${{ matrix.python-version }} on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
@@ -211,7 +213,7 @@ jobs:
           path: dist/torchdata*.whl
 
   wheel_upload:
-    if: always() && inputs.branch != ''
+    if: always() && inputs.branch != '' && inputs.do-upload == true
     needs: [get_release_type, wheel_build_test]
     runs-on: ubuntu-latest
     outputs:
@@ -289,31 +291,36 @@ jobs:
         with:
           ref: ${{ inputs.branch }}
           submodules: recursive
+      - name: Add temp runner environment variables
+        shell: bash -l {0}
+        run: |
+          echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}"
+          echo "CONDA_ENV_PATH=${RUNNER_TEMP}/conda_build_env" >> "${GITHUB_ENV}"
       - name: Create Conda Env
         if: ${{ ! startsWith( matrix.os, 'macos' ) }}
         uses: conda-incubator/setup-miniconda@v2
         with:
           python-version: ${{ matrix.python-version }}
-          activate-environment: conda_build_env
+          activate-environment: ${{ env.CONDA_ENV_PATH }}
       - name: Install Conda on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
         run: |
-          mkdir -p ~/miniconda3
+          mkdir -p "${MINICONDA_INSTALL_PATH_MACOS}"
           if ${{ startsWith( matrix.os, 'macos-m1' ) }}; then
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           else
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda3/miniconda.sh
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
           fi
-          bash ~/miniconda3/miniconda.sh -b -u -p ~/miniconda3
-          rm -rf ~/miniconda3/miniconda.sh
-          echo "$(dirname ~)/$(basename ~)/miniconda3/bin" >> $GITHUB_PATH
+          bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}"
+          rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh"
+          echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> $GITHUB_PATH
       - name: Create Conda Env on MacOS
         if: ${{ startsWith( matrix.os, 'macos' ) }}
         shell: bash -l {0}
         run: |
           conda init bash
-          conda create -y --name conda_build_env python=${{ matrix.python-version }}
+          conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }}
       - name: Setup additional system libraries
         if: startsWith( matrix.os, 'ubuntu' )
         run: |
@@ -338,14 +345,14 @@ jobs:
           BUILD_S3: ${{ steps.build_s3.outputs.value }}
         run: |
           set -ex
-          conda activate conda_build_env
+          conda activate "${CONDA_ENV_PATH}"
           conda install -yq conda-build -c conda-forge
           packaging/build_conda.sh
           conda index ./conda-bld
       - name: Install TorchData Conda Package
         shell: bash -l {0}
         run: |
-          conda activate conda_build_env
+          conda activate "${CONDA_ENV_PATH}"
           if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then
             CONDA_CHANNEL=pytorch
           else
@@ -355,7 +362,7 @@ jobs:
       - name: Run DataPipes Tests with pytest
         shell: bash -l {0}
         run: |
-          conda activate conda_build_env
+          conda activate "${CONDA_ENV_PATH}"
           pip3 install -r test/requirements.txt
           pytest --no-header -v test --ignore=test/test_period.py --ignore=test/test_text_examples.py --ignore=test/test_audio_examples.py
       - name: Upload Conda Package to Github
@@ -366,7 +373,7 @@ jobs:
           path: conda-bld/*/torchdata-*.tar.bz2
 
   conda_upload:
-    if: always() && inputs.branch != ''
+    if: always() && inputs.branch != '' && inputs.do-upload == true
     needs: [get_release_type, conda_build_test]
     runs-on: ubuntu-latest
     container: continuumio/miniconda3

.github/workflows/pull_release.yml

@@ -0,0 +1,21 @@
+name: Test Release Pipelines
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/pull_release.yml
+      - .github/workflows/_build_test_upload.yml
+
+jobs:
+  build_test_upload:
+    if: github.repository == 'pytorch/data'
+    uses: ./.github/workflows/_build_test_upload.yml
+    with:
+      branch: ""
+      pre_dev_release: true
+      pytorch_version: ""
+      do-upload: false
+    secrets:
+      PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ""
+      PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ""
+      CONDA_TEST_PYTORCHBOT_TOKEN: ""