Merge pull request #1406 from neagix/fix/access-restrict #1886

Workflow file for this run

.github/workflows/ci.yaml at 320245d

	name: CI

	on:
	push:
	branches:
	- dev
	- master
	pull_request: ~

	env:
	CACHE_VERSION: 1
	DEFAULT_PYTHON: "3.12"
	PRE_COMMIT_CACHE: ~/.cache/pre-commit
	DOCKER_BUILDKIT: 1
	COMPOSE_DOCKER_CLI_BUILD: 1
	BUILDKIT_PROGRESS: plain
	DOCKER_COMPOSE_CI_FILES: >-
	--file azure-pipelines/docker-compose-build.yaml
	--env-file azure-pipelines/.env

	jobs:
	prepare-python-venv:
	name: Prepare Python venv
	runs-on: ubuntu-24.04
	outputs:
	python-venv-cache-key: ${{ steps.python-venv-cache-key.outputs.key }}
	steps:
	# Create Python virtual env used for linters
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	id: python
	uses: actions/setup-python@v6
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	# Create a virtual environment and cache it
	- name: Generate Python virtual environment restore key
	id: python-venv-cache-key
	env:
	cache-name: cache-python-venv
	run: >-
	echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ steps.python.outputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
	- name: Restore Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ steps.python-venv-cache-key.outputs.key }}
	- name: Create Python virtual environment
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends \
	libcairo2-dev \
	libgirepository1.0-dev \
	pkg-config
	python3 -m venv --system-site-packages venv
	- name: Install requirements into Python virtual environment
	if: steps.cache-venv.outputs.cache-hit != 'true'
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	pip3 install --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pkgs.dev.azure.com/viseron/Viseron%20Pipelines/_packaging/viseron-wheels/pypi/simple -r requirements.txt -r requirements_test.txt -r requirements_ci.txt && \
	pre-commit install

	prepare-pre-commit:
	name: Prepare pre-commit
	runs-on: ubuntu-24.04
	needs: prepare-python-venv
	outputs:
	pre-commit-cache-key: ${{ steps.pre-commit-cache-key.outputs.key }}
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Generate pre-commit restore key
	id: pre-commit-cache-key
	env:
	cache-name: cache-pre-commit
	run: >-
	echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
	# Install pre-commit hooks into the cached virtual environment
	- name: Restore pre-commit hooks
	id: cache-pre-commit
	uses: actions/cache@v5
	with:
	path: ${{ env.PRE_COMMIT_CACHE }}
	key: ${{ steps.pre-commit-cache-key.outputs.key }}
	- name: Create pre-commit hooks
	if: steps.cache-pre-commit.outputs.cache-hit != 'true'
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	pre-commit install-hooks

	run-codespell:
	name: Run codespell
	runs-on: ubuntu-24.04
	needs: [prepare-python-venv, prepare-pre-commit]
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Restore pre-commit environment from cache
	id: cache-pre-commit
	uses: actions/cache@v5
	with:
	path: ${{ env.PRE_COMMIT_CACHE }}
	key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
	- name: Fail job if pre-commit cache restore failed
	if: steps.cache-pre-commit.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore pre-commit environment from cache"
	exit 1
	- name: Run codespell
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	pre-commit run --hook-stage manual codespell --all-files --show-diff-on-failure

	run-mypy:
	name: Run mypy
	runs-on: ubuntu-24.04
	needs: [prepare-python-venv, prepare-pre-commit]
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Restore pre-commit environment from cache
	id: cache-pre-commit
	uses: actions/cache@v5
	with:
	path: ${{ env.PRE_COMMIT_CACHE }}
	key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
	- name: Fail job if pre-commit cache restore failed
	if: steps.cache-pre-commit.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore pre-commit environment from cache"
	exit 1
	- name: Run mypy
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	mypy --install-types --non-interactive viseron
	pre-commit run --hook-stage manual mypy --all-files --show-diff-on-failure

	run-pylint:
	name: Run pylint
	runs-on: ubuntu-24.04
	needs: [prepare-python-venv, prepare-pre-commit]
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Restore pre-commit environment from cache
	id: cache-pre-commit
	uses: actions/cache@v5
	with:
	path: ${{ env.PRE_COMMIT_CACHE }}
	key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
	- name: Fail job if pre-commit cache restore failed
	if: steps.cache-pre-commit.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore pre-commit environment from cache"
	exit 1
	- name: Install dependencies
	run: \|
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends python3-gi python3-gst-1.0
	- name: Run pylint
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	pylint --version
	pylint viseron

	run-flake8:
	name: Run flake8
	runs-on: ubuntu-24.04
	needs: [prepare-python-venv, prepare-pre-commit]
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Restore pre-commit environment from cache
	id: cache-pre-commit
	uses: actions/cache@v5
	with:
	path: ${{ env.PRE_COMMIT_CACHE }}
	key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
	- name: Fail job if pre-commit cache restore failed
	if: steps.cache-pre-commit.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore pre-commit environment from cache"
	exit 1
	- name: Run flake8
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	pre-commit run --hook-stage manual flake8 --all-files --show-diff-on-failure

	run-pyupgrade:
	name: Run pyupgrade
	runs-on: ubuntu-24.04
	needs: [prepare-python-venv, prepare-pre-commit]
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Restore pre-commit environment from cache
	id: cache-pre-commit
	uses: actions/cache@v5
	with:
	path: ${{ env.PRE_COMMIT_CACHE }}
	key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
	- name: Fail job if pre-commit cache restore failed
	if: steps.cache-pre-commit.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore pre-commit environment from cache"
	exit 1
	- name: Run pyupgrade
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	pre-commit run --hook-stage manual pyupgrade --all-files --show-diff-on-failure

	# Run ruff, but dont fail yet since we are in the process of migrating
	run-ruff:
	name: Run ruff
	runs-on: ubuntu-24.04
	needs: [prepare-python-venv, prepare-pre-commit]
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Run ruff check
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	ruff --version
	ruff check \|\| true
	- name: Run ruff format check
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	ruff format --check \|\| true

	run-pytest:
	name: Run pytest
	runs-on: ubuntu-24.04
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	with:
	fetch-depth: 2
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	with:
	install: true
	- name: Build wheels Docker image
	run: \|
	docker compose ${{ env.DOCKER_COMPOSE_CI_FILES }} build amd64-wheels
	- name: Build pytest Docker image
	run: \|
	docker compose ${{ env.DOCKER_COMPOSE_CI_FILES }} build amd64-viseron-tests
	- name: Run pytest
	run: \|
	if docker compose ${{ env.DOCKER_COMPOSE_CI_FILES }} up --no-build amd64-viseron-tests; then
	exit 0
	else
	exit 1
	fi
	- name: Copy .coverage to host
	run: \|
	docker cp amd64-viseron-tests:/src/coverage.xml coverage.xml
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3

	run-generated-docs:
	name: Check generated docs
	runs-on: ubuntu-24.04
	needs: [prepare-python-venv, prepare-pre-commit]
	steps:
	- name: Check out code from GitHub
	uses: actions/checkout@v6
	- name: Set up Python ${{ env.DEFAULT_PYTHON }}
	uses: actions/setup-python@v6
	id: python
	with:
	python-version: ${{ env.DEFAULT_PYTHON }}
	- name: Restore base Python virtual environment
	id: cache-venv
	uses: actions/cache@v5
	with:
	path: venv
	key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
	- name: Fail job if Python cache restore failed
	if: steps.cache-venv.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore Python virtual environment from cache"
	exit 1
	- name: Restore pre-commit environment from cache
	id: cache-pre-commit
	uses: actions/cache@v5
	with:
	path: ${{ env.PRE_COMMIT_CACHE }}
	key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
	- name: Fail job if pre-commit cache restore failed
	if: steps.cache-pre-commit.outputs.cache-hit != 'true'
	run: \|
	echo "Failed to restore pre-commit environment from cache"
	exit 1
	- name: Install libedgetpu1 for edgetpu docs
	run: \|
	curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \| sudo apt-key add -
	sudo add-apt-repository "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main"
	sudo apt-get update
	sudo apt-get install -y --no-install-recommends libedgetpu1-std python3-gi python3-gst-1.0
	- name: Install libhailort for hailo docs
	run: \|
	set -e
	HAILO_VERSION=$(grep '^HAILO_VERSION=' azure-pipelines/.env \| cut -d'"' -f2)
	if [ -z "$HAILO_VERSION" ]; then
	echo "Failed to determine HAILO_VERSION" >&2
	exit 1
	fi
	UBUNTU_VERSION=$(grep '^UBUNTU_VERSION=' azure-pipelines/.env \| cut -d'"' -f2)
	if [ -z "$UBUNTU_VERSION" ]; then
	echo "Failed to determine UBUNTU_VERSION" >&2
	exit 1
	fi
	IMAGE_NAME="roflcoopter/amd64-hailo:${HAILO_VERSION}-${UBUNTU_VERSION}"
	echo "Pulling $IMAGE_NAME"
	docker pull $IMAGE_NAME
	CID=$(docker create "$IMAGE_NAME" bash)

	echo "Copying libhailort version $HAILO_VERSION from Docker image"
	sudo docker cp $CID:/usr/local/lib/libhailort.so.${HAILO_VERSION} /usr/local/lib/
	sudo ldconfig

	docker rm $CID
	echo "Extracted libhailort files:"; ls -1 /usr/local/lib/libhailort.*
	- name: Run script to check generated docs
	uses: ./.github/templates/run_in_venv
	with:
	command: \|
	pre-commit run --hook-stage manual generate_docs --all-files --show-diff-on-failure
	pre-commit run --hook-stage manual check_config_json --all-files --show-diff-on-failure

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Merge pull request #1406 from neagix/fix/access-restrict #1886

Workflow file

Merge pull request #1406 from neagix/fix/access-restrict #1886

Uh oh!

Workflow file for this run