[Docs] Add guidelines for contributing new functions or modules (#931) #648

Workflow file for this run

.github/workflows/test-all.yaml at 9f149c8

	name: Test all assets, build marketplace
	permissions:
	contents: read
	on:
	push:
	branches:
	- development
	- master
	workflow_dispatch: {}

	jobs:
	build_strategy_matrix:
	runs-on: ubuntu-latest
	steps:
	- name: Get the current branch name
	shell: bash
	run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
	id: myref

	- uses: actions/checkout@v4
	- id: set-matrix
	# This is very hacky, but it goes like that:
	# 1) Associate base_ref with origin/base_ref since actions/checkout doesn't do it, if we don't do that we won't be able to check the actual diff
	# 2) Build JSON string
	# 2.1) Add beginning of JSON
	# 2.2) Get diff between origin/base_ref and the checked-out repo => git diff ${{ github.base_ref }} --name-only
	# 2.3) Clean the file name and leave us only with directories => sed 's,/[^/]\+/$,,'
	# 2.4) Sort and keep only unique directories => sort \| uniq
	# 2.5) Remove directories starting with '.' => grep -v '^\.'
	# 2.6) Add quotation marks to all strings => sed 's/.*/"&"/'
	# 2.7) Add comma suffix to all strings excluding the last one => sed '$!s/.*/&,/'
	# 2.8) Close JSON
	# 3) Save matrix JSON to output
	# This is old fetch command it cant work cause base_ref is only avaliable on pull request actions: git fetch origin ${{ github.base_ref }}:${{ github.base_ref }}
	run: \|
	all_files="$(git ls-files \|\| true)"
	# Collect candidate package paths from diff
	candidates=$(
	printf '%s\n' "$all_files" \
	\| awk -F'/' '
	/^functions\/src\// {print $1"/"$2"/"$3}
	/^modules\/src\// {print $1"/"$2"/"$3}
	' \
	\| sort -u
	)

	# Keep only those that are actual directories
	packages=""
	for dir in $candidates; do
	if [[ -d "$dir" ]]; then
	packages+="$dir"$'\n'
	fi
	done

	if [[ -z "$packages" ]]; then
	matrix_json='{"package":[]}'
	else
	matrix_json=$(printf '%s\n' "$packages" \| grep -v '^$' \| jq -R . \| jq -s '{package: .}' \| jq -c)
	fi
	echo "matrix=$matrix_json" >> "$GITHUB_OUTPUT"
	outputs:
	matrix: ${{ steps.set-matrix.outputs.matrix }}

	check_matrix:
	runs-on: ubuntu-latest
	needs: build_strategy_matrix
	steps:
	- name: Install json2yaml
	run: \|
	sudo npm install -g json2yaml
	- name: Check matrix definition
	run: \|
	matrix='${{ needs.build_strategy_matrix.outputs.matrix }}'
	echo $matrix
	echo $matrix \| jq .
	echo $matrix \| json2yaml

	run_monorepo_tests:
	needs: build_strategy_matrix
	runs-on: ubuntu-latest
	strategy:
	# matrix: [{"package": some package that changed}, {...}, ...]
	matrix: ${{fromJson(needs.build_strategy_matrix.outputs.matrix)}}
	steps:
	# Source
	- name: Checkout current repo
	uses: actions/checkout@v4
	# Install python 3.10.17
	- name: Install python 3.10.17
	uses: actions/setup-python@v4
	with:
	python-version: 3.10.17
	# Install dependencies
	- uses: actions/cache@v3
	id: cache
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install requirements
	run: \|
	pip install --upgrade pip
	pip install -r requirements.txt

	- name: Run py tests
	run: python -m cli.cli run-tests -r ${{ matrix.package }} -s py -fn $(basename "${{ matrix.package }}")
	continue-on-error: true
	# - name: Run ipynb tests
	# run: python functions/cli/cli.py run-tests -r functions -s ipynb

	update_readmes:
	needs: build_strategy_matrix
	if: github.repository == 'mlrun/functions' \|\| github.repository == 'mlrun/hub-assets'
	runs-on: ubuntu-latest
	permissions:
	contents: write
	steps:
	- name: Get the current branch name
	shell: bash
	run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
	id: branch
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- name: Install python 3.10.17
	uses: actions/setup-python@v4
	with:
	python-version: 3.10.17
	- name: Install requirements
	run: \|
	pip install --upgrade pip
	pip install -r requirements.txt
	- name: Regenerate README tables
	env:
	CHANNEL: ${{ steps.branch.outputs.branch }}
	run: python -m cli.cli update-readme -c $CHANNEL --asset functions --asset modules
	- name: Commit & push (if changed)
	env:
	USERNAME: ${{ secrets.USERNAME }}
	USEREMAIL: ${{ secrets.USERMAIL }}
	run: \|
	if git diff --quiet; then
	echo "No README changes."
	exit 0
	fi
	git config --local user.name $USERNAME
	git config --local user.email $USEREMAIL
	git add functions/README.md modules/README.md \|\| true
	git commit -m "chore(readme): auto-update asset tables [skip ci]"
	git push

	build-marketplace:
	name: Build marketplace
	if: (github.repository == 'mlrun/functions' \|\| github.repository == 'mlrun/hub-assets')
	runs-on: ubuntu-latest
	needs: run_monorepo_tests
	continue-on-error: false

	steps:
	- name: Get the current branch name
	shell: bash
	run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
	id: branch
	- name: Checkout current repo
	uses: actions/checkout@v4
	- name: Checkout Marketplace
	uses: actions/checkout@v4
	with:
	repository: mlrun/marketplace
	path: marketplace
	- name: Install python 3.10.17
	uses: actions/setup-python@v4
	with:
	python-version: 3.10.17
	- name: Install requirements
	run: \|
	pip install --upgrade pip
	pip install -r requirements.txt
	- name: Build marketplace docs
	env:
	CHANNEL: ${{ steps.branch.outputs.branch }}
	run: \|
	cd marketplace
	pwd
	git pull origin
	cd ..
	python -m cli.cli build-marketplace -s ./functions/src -sn functions -m marketplace -c $CHANNEL -v -f
	python -m cli.cli build-marketplace -s ./modules/src -sn modules -m marketplace -c $CHANNEL -v -f
	## Uncomment the following lines if you want to upload the built marketplace as an artifact
	# - name: Upload built marketplace as artifact
	# uses: actions/upload-artifact@v4
	# with:
	# name: marketplace-build
	# path: marketplace/**
	- name: Publish marketplace release
	env:
	GITHUB_TOKEN: ${{ secrets.MARKETPLACE_ACCESS_TOKEN_V3 }}
	USERNAME: ${{ secrets.USERNAME }}
	USEREMAIL: ${{ secrets.USERMAIL }}
	REPO_PATH: marketplace
	BASE_REPO: mlrun
	BASE_BRANCH: master
	run: \|
	cd marketplace
	pwd
	COMMIT_SHA=$(git rev-parse --short "$GITHUB_SHA")
	echo "commit sha: $COMMIT_SHA"
	echo "github sha: $GITHUB_SHA"
	BRANCH_NAME=marketplace-doc-gen-$COMMIT_SHA
	REMOTE=https://$USERNAME:$GITHUB_TOKEN@github.com/$BASE_REPO/$REPO_PATH.git
	echo "Validating environment params...";
	[ -z "${GITHUB_TOKEN}" ] && {
	echo 'Missing input "GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}".';
	exit 1;
	};
	git config --local user.name $USERNAME
	git config --local user.email $USEREMAIL
	git branch --set-upstream-to origin/master
	git remote -v
	echo "1. Checking out [$BRANCH_NAME]..."
	git checkout -b $BRANCH_NAME
	echo "2. Checking out [$BASE_BRANCH]..."
	git checkout $BASE_BRANCH
	git pull
	echo "3. Checking out [$BRANCH_NAME]..."
	git checkout $BRANCH_NAME
	echo "3a. merging"
	git merge $BASE_BRANCH
	echo "3b. status"
	git status
	git status --ignored
	find . -type f \| xargs ls -artl
	echo "3b. add"
	git add --all
	git status
	git status --ignored
	echo "4. Commiting changes..."
	echo "4a. git rev-parse"
	git rev-parse --show-toplevel
	echo "4b. git commit"
	git commit -a -m "Automatically generated by github-worflow[bot] for commit: $COMMIT_SHA"
	git status
	git status --ignored
	# https://stackoverflow.com/questions/64270867/auth-error-trying-to-copy-a-repo-with-github-actions
	git config -l \| grep 'http\..*\.extraheader' \| cut -d= -f1 \| xargs -L1 git config --unset-all
	echo "Pushing [$BRANCH_NAME] to remote [$REMOTE]"
	git push -f $REMOTE $BRANCH_NAME
	echo "Submiting pull request..."
	gh pr create --title "Marketplace update from $BRANCH_NAME" --body "github-workflow" --base $BASE_BRANCH --head $BRANCH_NAME --repo $BASE_REPO/$REPO_PATH

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Docs] Add guidelines for contributing new functions or modules (#931) #648

Workflow file

[Docs] Add guidelines for contributing new functions or modules (#931) #648

Uh oh!

Workflow file for this run