[onnx_utils] updated pytorch due to security vulnerability (#968) #674
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test all assets, build marketplace | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches: | |
| - development | |
| - master | |
| workflow_dispatch: {} | |
| jobs: | |
| build_strategy_matrix: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get the current branch name | |
| shell: bash | |
| run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT | |
| id: myref | |
| - uses: actions/checkout@v4 | |
| - id: set-matrix | |
| # This is very hacky, but it goes like that: | |
| # 1) Associate base_ref with origin/base_ref since actions/checkout doesn't do it, if we don't do that we won't be able to check the actual diff | |
| # 2) Build JSON string | |
| # 2.1) Add beginning of JSON | |
| # 2.2) Get diff between origin/base_ref and the checked-out repo => git diff ${{ github.base_ref }} --name-only | |
| # 2.3) Clean the file name and leave us only with directories => sed 's,/*[^/]\+/*$,,' | |
| # 2.4) Sort and keep only unique directories => sort | uniq | |
| # 2.5) Remove directories starting with '.' => grep -v '^\.' | |
| # 2.6) Add quotation marks to all strings => sed 's/.*/"&"/' | |
| # 2.7) Add comma suffix to all strings excluding the last one => sed '$!s/.*/&,/' | |
| # 2.8) Close JSON | |
| # 3) Save matrix JSON to output | |
| # This is old fetch command it cant work cause base_ref is only avaliable on pull request actions: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }} | |
| run: | | |
| all_files="$(git ls-files || true)" | |
| # Collect candidate package paths from diff | |
| candidates=$( | |
| printf '%s\n' "$all_files" \ | |
| | awk -F'/' ' | |
| /^functions\/src\// {print $1"/"$2"/"$3} | |
| /^modules\/src\// {print $1"/"$2"/"$3} | |
| /^steps\/src\// {print $1"/"$2"/"$3} | |
| ' \ | |
| | sort -u | |
| ) | |
| # Keep only those that are actual directories | |
| packages="" | |
| for dir in $candidates; do | |
| if [[ -d "$dir" ]]; then | |
| packages+="$dir"$'\n' | |
| fi | |
| done | |
| if [[ -z "$packages" ]]; then | |
| matrix_json='{"package":[]}' | |
| else | |
| matrix_json=$(printf '%s\n' "$packages" | grep -v '^$' | jq -R . | jq -s '{package: .}' | jq -c) | |
| fi | |
| echo "matrix=$matrix_json" >> "$GITHUB_OUTPUT" | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| check_matrix: | |
| runs-on: ubuntu-latest | |
| needs: build_strategy_matrix | |
| steps: | |
| - name: Install json2yaml | |
| run: | | |
| sudo npm install -g json2yaml | |
| - name: Check matrix definition | |
| run: | | |
| matrix='${{ needs.build_strategy_matrix.outputs.matrix }}' | |
| echo $matrix | |
| echo $matrix | jq . | |
| echo $matrix | json2yaml | |
| run_monorepo_tests: | |
| needs: build_strategy_matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # matrix: [{"package": some package that changed}, {...}, ...] | |
| matrix: ${{fromJson(needs.build_strategy_matrix.outputs.matrix)}} | |
| steps: | |
| # Source | |
| - name: Checkout current repo | |
| uses: actions/checkout@v4 | |
| # Install python 3.10.17 | |
| - name: Install python 3.10.17 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.10.17 | |
| # Install dependencies | |
| - uses: actions/cache@v3 | |
| id: cache | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install requirements | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Run py tests | |
| run: python -m cli.cli run-tests -r ${{ matrix.package }} -s py -fn $(basename "${{ matrix.package }}") | |
| continue-on-error: true | |
| # - name: Run ipynb tests | |
| # run: python functions/cli/cli.py run-tests -r functions -s ipynb | |
| update_readmes: | |
| needs: build_strategy_matrix | |
| if: github.repository == 'mlrun/functions' || github.repository == 'mlrun/hub-assets' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Get the current branch name | |
| shell: bash | |
| run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT | |
| id: branch | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install python 3.10.17 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.10.17 | |
| - name: Install requirements | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Regenerate README tables | |
| env: | |
| CHANNEL: ${{ steps.branch.outputs.branch }} | |
| run: python -m cli.cli update-readme -c $CHANNEL --asset functions --asset modules --asset steps | |
| - name: Commit & push (if changed) | |
| env: | |
| USERNAME: ${{ secrets.USERNAME }} | |
| USEREMAIL: ${{ secrets.USERMAIL }} | |
| run: | | |
| if git diff --quiet; then | |
| echo "No README changes." | |
| exit 0 | |
| fi | |
| git config --local user.name $USERNAME | |
| git config --local user.email $USEREMAIL | |
| git add functions/README.md modules/README.md steps/README.md || true | |
| git commit -m "chore(readme): auto-update asset tables [skip ci]" | |
| git push | |
| build-marketplace: | |
| name: Build marketplace | |
| if: (github.repository == 'mlrun/functions' || github.repository == 'mlrun/hub-assets') | |
| runs-on: ubuntu-latest | |
| needs: run_monorepo_tests | |
| continue-on-error: false | |
| steps: | |
| - name: Get the current branch name | |
| shell: bash | |
| run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT | |
| id: branch | |
| - name: Checkout current repo | |
| uses: actions/checkout@v4 | |
| - name: Checkout Marketplace | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: mlrun/marketplace | |
| path: marketplace | |
| - name: Install python 3.10.17 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.10.17 | |
| - name: Install requirements | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Build marketplace docs | |
| env: | |
| CHANNEL: ${{ steps.branch.outputs.branch }} | |
| run: | | |
| cd marketplace | |
| pwd | |
| git pull origin | |
| cd .. | |
| python -m cli.cli build-marketplace -s ./functions/src -sn functions -m marketplace -c $CHANNEL -v -f | |
| python -m cli.cli build-marketplace -s ./modules/src -sn modules -m marketplace -c $CHANNEL -v -f | |
| python -m cli.cli build-marketplace -s ./steps/src -sn steps -m marketplace -c $CHANNEL -v -f | |
| ## Uncomment the following lines if you want to upload the built marketplace as an artifact | |
| # - name: Upload built marketplace as artifact | |
| # uses: actions/upload-artifact@v4 | |
| # with: | |
| # name: marketplace-build | |
| # path: marketplace/** | |
| - name: Publish marketplace release | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.MARKETPLACE_ACCESS_TOKEN_V3 }} | |
| USERNAME: ${{ secrets.USERNAME }} | |
| USEREMAIL: ${{ secrets.USERMAIL }} | |
| REPO_PATH: marketplace | |
| BASE_REPO: mlrun | |
| BASE_BRANCH: master | |
| run: | | |
| cd marketplace | |
| pwd | |
| COMMIT_SHA=$(git rev-parse --short "$GITHUB_SHA") | |
| echo "commit sha: $COMMIT_SHA" | |
| echo "github sha: $GITHUB_SHA" | |
| BRANCH_NAME=marketplace-doc-gen-$COMMIT_SHA | |
| REMOTE=https://$USERNAME:$GITHUB_TOKEN@github.com/$BASE_REPO/$REPO_PATH.git | |
| echo "Validating environment params..."; | |
| [ -z "${GITHUB_TOKEN}" ] && { | |
| echo 'Missing input "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}".'; | |
| exit 1; | |
| }; | |
| git config --local user.name $USERNAME | |
| git config --local user.email $USEREMAIL | |
| git branch --set-upstream-to origin/master | |
| git remote -v | |
| echo "1. Checking out [$BRANCH_NAME]..." | |
| git checkout -b $BRANCH_NAME | |
| echo "2. Checking out [$BASE_BRANCH]..." | |
| git checkout $BASE_BRANCH | |
| git pull | |
| echo "3. Checking out [$BRANCH_NAME]..." | |
| git checkout $BRANCH_NAME | |
| echo "3a. merging" | |
| git merge $BASE_BRANCH | |
| echo "3b. status" | |
| git status | |
| git status --ignored | |
| find . -type f | xargs ls -artl | |
| echo "3b. add" | |
| git add --all | |
| git status | |
| git status --ignored | |
| echo "4. Commiting changes..." | |
| echo "4a. git rev-parse" | |
| git rev-parse --show-toplevel | |
| echo "4b. git commit" | |
| git commit -a -m "Automatically generated by github-worflow[bot] for commit: $COMMIT_SHA" | |
| git status | |
| git status --ignored | |
| # https://stackoverflow.com/questions/64270867/auth-error-trying-to-copy-a-repo-with-github-actions | |
| git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | xargs -L1 git config --unset-all | |
| echo "Pushing [$BRANCH_NAME] to remote [$REMOTE]" | |
| git push -f $REMOTE $BRANCH_NAME | |
| echo "Submiting pull request..." | |
| gh pr create --title "Marketplace update from $BRANCH_NAME" --body "github-workflow" --base $BASE_BRANCH --head $BRANCH_NAME --repo $BASE_REPO/$REPO_PATH |