forked from datahub-project/datahub
-
Notifications
You must be signed in to change notification settings - Fork 0
162 lines (158 loc) · 7.94 KB
/
airflow-plugin.yml
File metadata and controls
162 lines (158 loc) · 7.94 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
name: Airflow Plugin
on:
push:
branches:
- master
- releases/**
paths:
- ".github/workflows/airflow-plugin.yml"
- "metadata-ingestion-modules/airflow-plugin/**"
- "metadata-ingestion/setup.py"
- "metadata-ingestion/pyproject.toml"
- "metadata-ingestion/constraints.txt"
- "metadata-ingestion/build.gradle"
- "metadata-ingestion/src/datahub/api/**"
- "metadata-ingestion/src/datahub/cli/**"
- "metadata-ingestion/src/datahub/configuration/**"
- "metadata-ingestion/src/datahub/emitter/**"
- "metadata-ingestion/src/datahub/ingestion/api/**"
- "metadata-ingestion/src/datahub/ingestion/graph/**"
- "metadata-ingestion/src/datahub/ingestion/sink/**"
- "metadata-ingestion/src/datahub/ingestion/source/sql/sqlalchemy_uri_mapper.py"
- "metadata-ingestion/src/datahub/metadata/**"
- "metadata-ingestion/src/datahub/sql_parsing/**"
- "metadata-ingestion/src/datahub/telemetry/**"
- "metadata-ingestion/src/datahub/testing/**"
- "metadata-ingestion/src/datahub/utilities/**"
- "metadata-models/**"
pull_request:
branches:
- "**"
paths:
- ".github/workflows/airflow-plugin.yml"
- "metadata-ingestion-modules/airflow-plugin/**"
- "metadata-ingestion/setup.py"
- "metadata-ingestion/pyproject.toml"
- "metadata-ingestion/constraints.txt"
- "metadata-ingestion/build.gradle"
- "metadata-ingestion/src/datahub/api/**"
- "metadata-ingestion/src/datahub/cli/**"
- "metadata-ingestion/src/datahub/configuration/**"
- "metadata-ingestion/src/datahub/emitter/**"
- "metadata-ingestion/src/datahub/ingestion/api/**"
- "metadata-ingestion/src/datahub/ingestion/graph/**"
- "metadata-ingestion/src/datahub/ingestion/sink/**"
- "metadata-ingestion/src/datahub/ingestion/source/sql/sqlalchemy_uri_mapper.py"
- "metadata-ingestion/src/datahub/metadata/**"
- "metadata-ingestion/src/datahub/sql_parsing/**"
- "metadata-ingestion/src/datahub/telemetry/**"
- "metadata-ingestion/src/datahub/testing/**"
- "metadata-ingestion/src/datahub/utilities/**"
- "metadata-models/**"
release:
types: [published]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
DEPOT_PROJECT_ID: "${{ vars.DEPOT_PROJECT_ID }}"
jobs:
airflow-plugin:
runs-on: ${{ vars.DEPOT_PROJECT_ID != '' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
env:
DATAHUB_TELEMETRY_ENABLED: false
strategy:
matrix:
include:
# Note: this should be kept in sync with tox.ini.
# Airflow 2.x versions use standalone openlineage-airflow package (airflow2)
# Airflow 3.x versions use native OpenLineage provider (airflow3)
#
# Known limitation: Airflow constraints pin jaraco.context to older versions
# (e.g., 5.3.0 in 2.9.x) which conflicts with acryl-datahub's requirement of
# jaraco.context>=6.1.0 (security fix for GHSA-58pv-8j8x-9vj2). This is acceptable
# for CI testing; the security constraint is enforced in Docker image builds via
# docker/snippets/ingestion/constraints.txt.
#
# Similarly, Airflow constraints pin aiohttp to older versions (e.g., 3.10.10 in 2.x,
# 3.12.15 in 3.x) which conflicts with acryl-datahub's requirement of aiohttp>=3.13.3
# (CVE-2025-30304, CVE-2025-32442). This is acceptable for CI testing; the security
# constraint is enforced in Docker image builds via docker/snippets/ingestion/constraints.txt.
- python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.7.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.3/constraints-3.10.txt"
extra_pip_extras: "airflow2"
- python-version: "3.10"
extra_pip_requirements: "apache-airflow~=2.8.1"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.8.1/constraints-3.10.txt"
extra_pip_extras: "airflow2"
- python-version: "3.11"
extra_pip_requirements: "apache-airflow~=2.9.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.9.3/constraints-3.11.txt"
extra_pip_extras: "airflow2"
- python-version: "3.11"
extra_pip_requirements: "apache-airflow~=2.10.3"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
extra_pip_extras: "airflow2"
# Test Airflow 2.10 with apache-airflow-providers-openlineage instead of openlineage-airflow
- python-version: "3.11"
extra_pip_requirements: "apache-airflow~=2.10.3 apache-airflow-providers-openlineage>=1.0.0"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-2.10.3/constraints-3.11.txt"
extra_pip_extras: ""
- python-version: "3.11"
extra_pip_requirements: "apache-airflow~=3.1.0 apache-airflow-providers-openlineage>=2.8.0"
extra_pip_constraints: "-c https://raw.githubusercontent.com/apache/airflow/constraints-3.1.0/constraints-3.11.txt"
extra_pip_extras: "airflow3"
fail-fast: false
steps:
- name: Set up JDK 21
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
with:
distribution: "zulu"
java-version: 21
- uses: gradle/actions/setup-gradle@ed408507eac070d1f99cc633dbcf757c94c7933a # v4
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
- name: Install airflow package and test (extras ${{ matrix.extra_pip_requirements }})
run: ./gradlew -Pextra_pip_requirements='${{ matrix.extra_pip_requirements }}' -Pextra_pip_constraints='${{ matrix.extra_pip_constraints }}' -Pextra_pip_extras='${{ matrix.extra_pip_extras }}' :metadata-ingestion-modules:airflow-plugin:build
- name: pip freeze show list installed
if: always()
run: source metadata-ingestion-modules/airflow-plugin/venv/bin/activate && uv pip freeze
- name: Report test results
if: (!cancelled())
uses: ./.github/actions/report-test-results
with:
artifact-name: ingestion-airflow-plugin-test-results-${{ strategy.job-index }}
test-results-paths: "metadata-ingestion-modules/airflow-plugin/test-results/**"
junit-file-globs: "metadata-ingestion-modules/airflow-plugin/test-results/*.xml"
- name: Upload coverage to Codecov with ingestion flag
if: (!cancelled())
uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
directory: ./build/coverage-reports/metadata-ingestion-modules/airflow-plugin/
fail_ci_if_error: false
flags: ingestion-airflow
name: pytest-airflow-${{ strategy.job-index }}
verbose: true
override_branch: ${{ github.head_ref || github.ref_name }}
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
override_branch: ${{ github.head_ref || github.ref_name }}
event-file:
runs-on: ubuntu-latest
steps:
- name: Upload
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: Event File
path: ${{ github.event_path }}