Skip to content

feat(incidents): optional caller-provided id for idempotent raiseIncident #9700

feat(incidents): optional caller-provided id for idempotent raiseIncident

feat(incidents): optional caller-provided id for idempotent raiseIncident #9700

Workflow file for this run

name: lint
on:
# Only run on PRs. Assuming all commits come in via PRs, there is no need for lint post commit
pull_request:
# Run only on below branches
branches:
- master
- releases/**
- hotfixes/**
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
setup:
name: Initial Job Setup ${{ github.workflow }}
runs-on: ubuntu-slim
outputs:
lint-job: ${{ steps.filter.outputs.lint-job }} # Run all lint jobs when this file is changed
markdown: ${{ steps.filter.outputs.markdown == 'true' || steps.filter.outputs.lint-job == 'true' }}
github-actions: ${{ steps.filter.outputs.github-actions == 'true' || steps.filter.outputs.lint-job == 'true' }}
workflow-files: ${{ steps.filter.outputs.workflow-files == 'true' || steps.filter.outputs.lint-job == 'true' }}
workflow-validation: ${{ steps.filter.outputs.workflow-validation == 'true' || steps.filter.outputs.lint-job == 'true' }}
code-check-sources: ${{ steps.filter.outputs.code-check-sources == 'true' || steps.filter.outputs.lint-job == 'true' }}
pdl: ${{ steps.filter.outputs.pdl == 'true' || steps.filter.outputs.lint-job == 'true' }}
datahub-web-react: ${{ steps.filter.outputs.datahub-web-react == 'true' || steps.filter.outputs.lint-job == 'true' }}
metadata-ingestion: ${{ steps.filter.outputs.metadata-ingestion == 'true' || steps.filter.outputs.lint-job == 'true' }}
datahub-actions: ${{ steps.filter.outputs.datahub-actions == 'true' || steps.filter.outputs.lint-job == 'true' }}
datahub-agent-context: ${{ steps.filter.outputs.datahub-agent-context == 'true' || steps.filter.outputs.lint-job == 'true' }}
airflow-plugin: ${{ steps.filter.outputs.airflow-plugin == 'true' || steps.filter.outputs.lint-job == 'true' }}
gx-plugin: ${{ steps.filter.outputs.gx-plugin == 'true' || steps.filter.outputs.lint-job == 'true' }}
dagster-plugin: ${{ steps.filter.outputs.dagster-plugin == 'true' || steps.filter.outputs.lint-job == 'true' }}
prefect-plugin: ${{ steps.filter.outputs.prefect-plugin == 'true' || steps.filter.outputs.lint-job == 'true' }}
# Runs the single python-lint job if ANY Python module's files changed.
python-lint: ${{ steps.filter.outputs.metadata-ingestion == 'true' || steps.filter.outputs.datahub-actions == 'true' || steps.filter.outputs.datahub-agent-context == 'true' || steps.filter.outputs.airflow-plugin == 'true' || steps.filter.outputs.gx-plugin == 'true' || steps.filter.outputs.dagster-plugin == 'true' || steps.filter.outputs.prefect-plugin == 'true' || steps.filter.outputs.lint-job == 'true' }}
smoke-test-python: ${{ steps.filter.outputs.smoke-test-python == 'true' || steps.filter.outputs.lint-job == 'true' }}
smoke-test-cypress: ${{ steps.filter.outputs.smoke-test-cypress == 'true' || steps.filter.outputs.lint-job == 'true' }}
playwright-e2e: ${{ steps.filter.outputs.playwright-e2e == 'true' || steps.filter.outputs.lint-job == 'true' }}
default-runner: ${{ steps.runners.outputs.default-runner }}
default-gh-runner: ${{ steps.runners.outputs.default-gh-runner }}
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4
id: filter
with:
filters: .github/path-filters/lint-jobs-filters.yml
- uses: ./.github/actions/determine-runners
id: runners
with:
is-community-pr: ${{ github.event.pull_request.head.repo.full_name != github.repository }}
has-depot-label: ${{ contains(github.event.pull_request.labels.*.name, 'depot') }}
markdown-format:
name: markdown_format_check
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.markdown == 'true'
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
with:
distribution: "zulu"
java-version: 21
- uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2
# mdPrettierCheck dependsOn yarnInstall (full datahub-web-react deps).
# Cache ~/.cache/yarn so the install is a warm no-op instead of a cold fetch.
- name: Restore yarn cache
id: yarn-cache
uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
with:
path: ~/.cache/yarn
key: ${{ runner.os }}-yarn-${{ hashFiles('datahub-web-react/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: run markdown checks
run: ./gradlew :datahub-web-react:mdPrettierCheck :datahub-web-react:docusaurusMarkdownLint
- name: Save yarn cache
if: always() && steps.yarn-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
with:
path: ~/.cache/yarn
key: ${{ runner.os }}-yarn-${{ hashFiles('datahub-web-react/yarn.lock') }}
github-actions-format:
name: github_actions_format_check
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.github-actions == 'true'
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
with:
distribution: "zulu"
java-version: 21
- uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2
# githubActionsPrettierCheck dependsOn yarnInstall (full datahub-web-react deps).
# Cache ~/.cache/yarn so the install is a warm no-op instead of a cold fetch.
- name: Restore yarn cache
id: yarn-cache
uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
with:
path: ~/.cache/yarn
key: ${{ runner.os }}-yarn-${{ hashFiles('datahub-web-react/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: run prettier --check
run: ./gradlew :datahub-web-react:githubActionsPrettierCheck
- name: Save yarn cache
if: always() && steps.yarn-cache.outputs.cache-hit != 'true'
uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
with:
path: ~/.cache/yarn
key: ${{ runner.os }}-yarn-${{ hashFiles('datahub-web-react/yarn.lock') }}
validate-post-workflow-list:
name: validate_post_workflow_list
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.workflow-validation == 'true'
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.10"
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Install pyyaml
run: uv pip install pyyaml --system
- name: Validate post-workflow-actions.yml workflow list
run: python .github/scripts/validate_post_workflow_list.py
- name: Write step summary
if: always()
run: cat post-workflow-validation-summary.md >> "$GITHUB_STEP_SUMMARY"
actionlint:
name: actionlint
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.workflow-files == 'true'
permissions:
contents: read
checks: write
pull-requests: write
issues: write
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: reviewdog/action-actionlint@0d952c597ef8459f634d7145b0b044a9699e5e43 # v1
with:
reporter: github-pr-review
check-event-types:
name: check-event-types
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.code-check-sources == 'true'
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.10"
- name: run check
run: python .github/scripts/check_event_type.py
check-policies:
name: check-policies
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.code-check-sources == 'true'
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.10"
- name: run check
run: python .github/scripts/check_policies.py
check-gradle-lockfiles:
name: check-gradle-lockfiles
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.code-check-sources == 'true'
steps:
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.10"
- name: run check
run: python .github/scripts/check_gradle_lockfiles.py
check-schema-versions:
name: check-schema-versions
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.pdl == 'true'
steps:
# Full history so the script can compute the merge-base against the PR's
# base branch and diff the changed PDL aspects against it.
- uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
with:
fetch-depth: 0
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.11"
- name: Verify schemaVersion is bumped for changed PDL aspects
env:
BASE_REF: ${{ github.base_ref }}
run: python .github/scripts/bump_schema_versions.py --check --base-branch "$BASE_REF"
smoke-test-lint:
name: Lint on smoke tests
runs-on: ${{ needs.setup.outputs.default-runner }}
needs: setup
if: ${{ needs.setup.outputs.smoke-test-python == 'true' }}
env:
UV_CACHE_DIR: /tmp/.uv-cache/
YARN_CACHE_FOLDER: /tmp/.yarn-cache/
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.11"
cache: "pip"
- uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5
# Restore-only: this workflow is pull_request-only, so a save here would be
# ref-scoped to the PR and never shared. The master-scoped entry is written by
# ci-cache-warmup.yml (push to master) under this exact key.
# Key must stay byte-identical with the lookup and save keys in
# ci-cache-warmup.yml warm-smoke-test-lint-cache. metadata-ingestion's
# manifests are included because requirements.txt installs '-e ../metadata-ingestion[...]'.
- uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
name: Restore uv cache
if: ${{ needs.setup.outputs.smoke-test-python == 'true' }}
with:
path: |
${{ env.UV_CACHE_DIR }}
key: "uv-smoke-test-lint-${{ hashFiles('./smoke-test/requirements.txt', './smoke-test/pyproject.toml', './metadata-ingestion/setup.py', './metadata-ingestion/pyproject.toml') }}"
restore-keys: |
uv-smoke-test-lint-
- name: Run Python lint
if: ${{ needs.setup.outputs.smoke-test-python == 'true' }}
run: |
python ./.github/scripts/check_python_package.py
./gradlew :smoke-test:pythonLint
python-lint:
name: python-lint
runs-on: ${{ needs.setup.outputs.default-runner }}
needs: setup
if: needs.setup.outputs.python-lint == 'true'
env:
UV_CACHE_DIR: /tmp/.uv-cache/
DATAHUB_TELEMETRY_ENABLED: false
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
- name: Set up JDK 21
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5
with:
distribution: "zulu"
java-version: 21
- uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
with:
python-version: "3.11"
- name: Restore uv cache
uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
with:
path: ${{ env.UV_CACHE_DIR }}
# Restore-only: Actions caches are ref-scoped, so a PR-side save can never be
# shared with other PRs — it just mints a per-PR copy against the repo's 10GB
# budget. The master-scoped entry every PR restores from is written by
# ci-cache-warmup.yml (push to master), which uses this exact key.
# Key must stay byte-identical with the lookup and save keys in
# ci-cache-warmup.yml warm-python-lint-cache (hashFiles cannot be
# deduplicated across workflow files).
key: uv-python-lint-${{ hashFiles('./metadata-ingestion/setup.py', './metadata-ingestion/pyproject.toml', './metadata-ingestion/uv.lock', './metadata-ingestion/constraints.txt', './metadata-ingestion/build-constraints.txt', './datahub-actions/setup.py', './datahub-actions/pyproject.toml', './datahub-agent-context/setup.py', './datahub-agent-context/pyproject.toml', './metadata-ingestion-modules/airflow-plugin/setup.py', './metadata-ingestion-modules/airflow-plugin/pyproject.toml', './metadata-ingestion-modules/gx-plugin/setup.py', './metadata-ingestion-modules/gx-plugin/pyproject.toml', './metadata-ingestion-modules/dagster-plugin/setup.py', './metadata-ingestion-modules/dagster-plugin/pyproject.toml', './metadata-ingestion-modules/prefect-plugin/setup.py', './metadata-ingestion-modules/prefect-plugin/pyproject.toml') }}
restore-keys: |
uv-python-lint-
- name: Install system dependencies
run: ./metadata-ingestion/scripts/install_deps.sh
# Per-module lint steps. Each runs independently (if: always()) so a failure in one
# does not skip the others. Each uses its module's own Gradle-managed venv (separate
# venvs) while sharing the single uv download cache above. Each is gated by its own
# path-filter output.
- name: metadata-ingestion lint
id: metadata-ingestion-lint
if: always() && needs.setup.outputs.metadata-ingestion == 'true'
run: ./gradlew :metadata-ingestion:lint
- name: datahub-actions lint
id: datahub-actions-lint
if: always() && needs.setup.outputs.datahub-actions == 'true'
run: ./gradlew :datahub-actions:lint
- name: datahub-agent-context lint
id: datahub-agent-context-lint
if: always() && needs.setup.outputs.datahub-agent-context == 'true'
run: ./gradlew :datahub-agent-context:lint
- name: airflow-plugin lint
id: airflow-plugin-lint
if: always() && needs.setup.outputs.airflow-plugin == 'true'
run: ./gradlew :metadata-ingestion-modules:airflow-plugin:lint
- name: gx-plugin lint
id: gx-plugin-lint
if: always() && needs.setup.outputs.gx-plugin == 'true'
run: ./gradlew :metadata-ingestion-modules:gx-plugin:lint
- name: dagster-plugin lint
id: dagster-plugin-lint
if: always() && needs.setup.outputs.dagster-plugin == 'true'
run: ./gradlew :metadata-ingestion-modules:dagster-plugin:lint
- name: prefect-plugin lint
id: prefect-plugin-lint
if: always() && needs.setup.outputs.prefect-plugin == 'true'
run: ./gradlew :metadata-ingestion-modules:prefect-plugin:lint
- name: Report lint results
if: always()
env:
MI: ${{ steps.metadata-ingestion-lint.outcome }}
DA: ${{ steps.datahub-actions-lint.outcome }}
DAC: ${{ steps.datahub-agent-context-lint.outcome }}
AF: ${{ steps.airflow-plugin-lint.outcome }}
GX: ${{ steps.gx-plugin-lint.outcome }}
DAG: ${{ steps.dagster-plugin-lint.outcome }}
PF: ${{ steps.prefect-plugin-lint.outcome }}
run: |
failed=()
[ "$MI" = "failure" ] && failed+=("metadata-ingestion")
[ "$DA" = "failure" ] && failed+=("datahub-actions")
[ "$DAC" = "failure" ] && failed+=("datahub-agent-context")
[ "$AF" = "failure" ] && failed+=("airflow-plugin")
[ "$GX" = "failure" ] && failed+=("gx-plugin")
[ "$DAG" = "failure" ] && failed+=("dagster-plugin")
[ "$PF" = "failure" ] && failed+=("prefect-plugin")
if [ ${#failed[@]} -gt 0 ]; then
echo "::error::Python lint failed for: ${failed[*]}"
exit 1
fi
echo "All Python lint steps passed (or were skipped)."
datahub-web-react-lint:
name: datahub-web-react-lint
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.datahub-web-react == 'true'
permissions:
contents: read
pull-requests: write # post the i18n parity summary as a sticky PR comment
env:
# Warn-only for now: reports missing/stale/placeholder drift without failing the build.
# Set to "fail" to make errors (stale keys, placeholder drift) block CI.
I18N_PARITY_MODE: warn
I18N_PARITY_SUMMARY_FILE: ${{ github.workspace }}/i18n-parity-summary.md
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
# Restore-only, from yarn's default cache dir: the master-scoped entry under this
# key is written by ci-cache-warmup.yml warm-web-yarn-cache (build-and-test's
# frontend jobs run on Depot runners, whose cache writes GitHub-hosted jobs cannot
# read). Saving here would only mint a per-PR copy (this workflow is
# pull_request-only), so we never save.
- uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
name: Restore yarn cache
with:
path: |
~/.cache/yarn
key: ${{ runner.os }}-yarn-${{ hashFiles('datahub-web-react/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5
- name: Run lint
run: |
./gradlew :datahub-web-react:yarnLint
- name: Check i18n locale parity
run: |
./gradlew :datahub-web-react:yarnI18nParityCheck
# Surface findings directly on the PR (not just in job logs). The summary file only exists when
# there are findings, so hashFiles() drives post-vs-delete. Skipped on fork PRs, where the
# GITHUB_TOKEN is read-only. always() so it still runs when the check fails (fail mode).
- name: Comment i18n parity summary on PR
if: always() && github.event.pull_request.head.repo.full_name == github.repository && hashFiles('i18n-parity-summary.md') != ''
uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3.0.2
with:
header: i18n-parity
path: i18n-parity-summary.md
- name: Clear i18n parity comment when in sync
if: always() && github.event.pull_request.head.repo.full_name == github.repository && hashFiles('i18n-parity-summary.md') == ''
uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987 # v3.0.2
with:
header: i18n-parity
delete: true
playwright-e2e-lint:
name: playwright-e2e-lint
runs-on: ${{ needs.setup.outputs.default-gh-runner }}
needs: setup
if: needs.setup.outputs.playwright-e2e == 'true'
steps:
- name: Check out the repo
uses: acryldata/sane-checkout-action@186e92cc5948a9c3e1cc7a96eaff9f776f3fc8e3 # v7
# Restore-only, from yarn's default cache dir, using the same key family as
# resusable-playwright-tests.yml (key must stay byte-identical with it and with
# ci-cache-warmup.yml warm-playwright-yarn-cache, whose GitHub-hosted save is
# what PR jobs here can actually read). The playwright- prefix keeps this small
# entry out of the '<os>-yarn-' family, so a web-cache prefix restore can never
# land on it; the secondary restore-key still falls back to the datahub-web-react
# yarn cache, which contains most of the lint toolchain (eslint, prettier,
# typescript). Saving here would only mint a per-PR copy (this workflow is
# pull_request-only), so we never save.
- uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
name: Restore yarn cache
with:
path: |
~/.cache/yarn
key: playwright-${{ runner.os }}-yarn-${{ hashFiles('./e2e-test/ui/playwright/package.json', './e2e-test/ui/playwright/yarn.lock') }}
restore-keys: |
playwright-${{ runner.os }}-yarn-
${{ runner.os }}-yarn-
- name: Set up Node.js
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: "22"
- name: Install dependencies
working-directory: e2e-test/ui/playwright
run: yarn install --frozen-lockfile
- name: Prettier format check
working-directory: e2e-test/ui/playwright
run: yarn format:check
- name: ESLint
working-directory: e2e-test/ui/playwright
run: yarn lint