feat: Add YAML custom connector publishing (Docker stubbed) #2497

Workflow file for this run

.github/workflows/python_pytest.yml at fea8293

	# This workflow will run pytest.
	#
	# There are three jobs which run in parallel:
	# 1. pytest-fast: Run fast tests only, and fail fast so the dev knows asap if they broke something.
	# 2. pytest-no-creds: Run tests only if they don't require creds. The main use case is to run tests
	# on forks, where secrets are not available. We flush the GCP_GSM_CREDENTIALS env var to an
	# invalid value to ensure that tests that require creds are not run.
	# 3. pytest: Run all tests, across multiple python versions.
	#
	# Note that pytest-fast also skips tests that require credentials, allowing it to run on forks.
	name: Run Tests

	on:
	push:
	branches:
	- main
	pull_request: {}

	env:
	AIRBYTE_ANALYTICS_ID: ${{ vars.AIRBYTE_ANALYTICS_ID }}

	jobs:
	pytest-fast:
	name: Pytest (Fast)
	runs-on: ubuntu-latest
	steps:
	# Common steps:
	- name: Checkout code
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	- name: Set up Poetry
	uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9
	with:
	poetry-version: "2.2.0"
	- name: Set up Python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: '3.10'
	- name: Set up Poetry
	uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9
	with:
	poetry-version: "2.2.0"
	- name: Install dependencies
	run: poetry install

	- name: Run Pytest with Coverage (Fast Tests Only)
	timeout-minutes: 60
	env:
	GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
	run: >
	poetry run coverage run -m pytest
	--durations=5 --exitfirst
	-m "not slow and not requires_creds and not linting and not flaky"

	- name: Print Coverage Report
	if: always()
	run: poetry run coverage report

	- name: Create Coverage Artifacts
	if: always()
	run: \|
	poetry run coverage html -d htmlcov
	poetry run coverage xml -o htmlcov/coverage.xml

	- name: Upload coverage to GitHub Artifacts
	if: always()
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	with:
	name: fasttest-coverage
	path: htmlcov/

	- name: Upload logs to GitHub Artifacts
	if: failure()
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	with:
	name: pytest-fast-test-logs
	path: /tmp/airbyte/logs/

	- name: Publish Test Results
	uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0
	if: always() && !cancelled()
	continue-on-error: true
	with:
	check_name: "PyTest Results (Fast Tests Only, No Creds)"
	large_files: true
	files: \|
	build/test-results/*/.xml

	pytest-no-creds:
	name: Pytest (No Creds)
	runs-on: ubuntu-latest
	steps:
	# Common steps:
	- name: Checkout code
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	- name: Set up Poetry
	uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9
	with:
	poetry-version: "2.2.0"
	- name: Set up Python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: '3.10'
	- name: Set up Poetry
	uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9
	with:
	poetry-version: "2.2.0"
	- name: Install dependencies
	run: poetry install

	# Job-specific step(s):
	- name: Run Pytest (No-Creds)
	timeout-minutes: 60
	env:
	# Force this to a blank value.
	GCP_GSM_CREDENTIALS: ""
	run: >
	poetry run coverage run -m pytest
	--durations=5 --exitfirst
	-m "not requires_creds and not linting and not super_slow and not flaky"

	- name: Print Coverage Report
	if: always()
	run: poetry run coverage report

	- name: Create Coverage Artifacts
	if: always()
	run: \|
	poetry run coverage html -d htmlcov
	poetry run coverage xml -o htmlcov/coverage.xml

	- name: Upload coverage to GitHub Artifacts
	if: always()
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	with:
	name: nocreds-test-coverage
	path: htmlcov/

	- name: Upload logs to GitHub Artifacts
	if: failure()
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	with:
	name: pytest-no-creds-test-logs
	path: /tmp/airbyte/logs/

	pytest:
	name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }})
	# Don't run on forks. Run on pushes to main, and on PRs that are not from forks.
	if: >
	(github.event_name == 'push' && github.ref == 'refs/heads/main') \|\|
	(github.event.pull_request.head.repo.fork == false)
	strategy:
	matrix:
	python-version: [
	'3.10',
	'3.11',
	# '3.12', # Commented out: Certain tests don't work in Python 3.12, although PyAirbyte itself does work on 3.12
	]
	os: [
	Ubuntu,
	Windows,
	]
	fail-fast: false

	runs-on: "${{ matrix.os }}-latest"
	env:
	# Enforce UTF-8 encoding so Windows runners don't fail inside the connector code.
	# TODO: See if we can fully enforce this within PyAirbyte itself.
	PYTHONIOENCODING: utf-8
	steps:
	# Common steps:
	- name: Checkout code
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	- name: Set up Python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: ${{ matrix.python-version }}
	- name: Set up Poetry
	uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9
	with:
	poetry-version: "2.2.0"
	- name: Install dependencies
	run: poetry install

	# Job-specific step(s):
	- name: Run Pytest [all]
	if: matrix.os != 'Windows'
	timeout-minutes: 60
	env:
	GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
	run: >
	poetry run coverage run -m pytest
	--verbose
	-m "not linting and not super_slow and not flaky"

	- name: Run Pytest [unit tests only]
	if: matrix.os == 'Windows'
	timeout-minutes: 60
	env:
	GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
	run: >
	poetry run coverage run -m pytest
	--verbose
	tests/unit_tests/
	-m "not linting and not super_slow and not flaky"

	- name: Print Coverage Report
	if: always()
	run: poetry run coverage report

	- name: Create Coverage Artifacts
	if: always()
	run: \|
	poetry run coverage html -d htmlcov
	poetry run coverage xml -o htmlcov/coverage.xml

	- name: Upload coverage to GitHub Artifacts
	if: always()
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	with:
	name: py${{ matrix.python-version }}-${{ matrix.os }}-test-coverage
	path: htmlcov/

	- name: Upload logs to GitHub Artifacts
	if: failure()
	uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
	with:
	name: py${{ matrix.python-version }}-${{ matrix.os }}-test-logs
	path: /tmp/airbyte/logs/

	- name: Publish Test Results
	uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 # v2.20.0
	if: always() && !cancelled() && matrix.python-version == '3.10' && matrix.os == 'Ubuntu'
	continue-on-error: true
	with:
	check_name: "PyTest Results (Full)"
	large_files: true
	files: \|
	build/test-results/*/.xml

	dependency-analysis:
	name: Dependency Analysis with Deptry
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	- name: Set up Python
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: '3.10'
	- name: Set up Poetry
	uses: Gr1N/setup-poetry@48b0f77c8c1b1b19cb962f0f00dff7b4be8f81ec # v9
	with:
	poetry-version: "2.2.0"
	- name: Install dependencies
	run: poetry install

	# Job-specific step(s):
	- name: Run Deptry
	run: \|
	poetry run deptry .

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

feat: Add YAML custom connector publishing (Docker stubbed) #2497

Workflow file

feat: Add YAML custom connector publishing (Docker stubbed) #2497

Uh oh!

Jobs

Run details

Workflow file for this run