ci: validate maintained examples

Author: ayush-shah

.github/dependabot.yml

@@ -0,0 +1,86 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /custom-connector
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+    groups:
+      custom-connector-python:
+        patterns: ["*"]
+
+  - package-ecosystem: pip
+    directory: /custom-graphql
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+    groups:
+      custom-graphql-python:
+        patterns: ["*"]
+
+  - package-ecosystem: pip
+    directory: /dynamic_csv_importer
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+    groups:
+      dynamic-csv-python:
+        patterns: ["*"]
+
+  - package-ecosystem: pip
+    directory: /api-lineage-cicd
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+    groups:
+      api-lineage-python:
+        patterns: ["*"]
+
+  - package-ecosystem: pip
+    directory: /ingestion-automation
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+    groups:
+      ingestion-automation-python:
+        patterns: ["*"]
+
+  - package-ecosystem: npm
+    directory: /custom-graphql/server
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+    groups:
+      graphql-server:
+        patterns: ["*"]
+
+  - package-ecosystem: docker
+    directory: /custom-connector/docker
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+      day: monday
+      time: "06:00"
+      timezone: Etc/UTC
+    groups:
+      github-actions:
+        patterns: ["*"]

.github/workflows/validate-examples.yml

@@ -0,0 +1,151 @@
+name: Validate maintained examples
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+  schedule:
+    - cron: "0 6 * * 1"
+
+permissions:
+  contents: read
+
+jobs:
+  python:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: |
+            custom-connector/pyproject.toml
+            custom-graphql/pyproject.toml
+            dynamic_csv_importer/requirements.txt
+            api-lineage-cicd/requirements.txt
+            ingestion-automation/requirements.txt
+
+      - name: Install exact OpenMetadata RC and test dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install --editable './custom-connector[test]'
+          python -m pip install --editable ./custom-graphql
+          python -m pip install \
+            --requirement dynamic_csv_importer/requirements.txt \
+            --requirement api-lineage-cicd/requirements.txt \
+            --requirement ingestion-automation/requirements.txt
+
+      - name: Verify the installed OpenMetadata version
+        run: |
+          python - <<'PY'
+          from importlib.metadata import version
+
+          assert version("openmetadata-ingestion") == "2.0.0.0rc1"
+          PY
+
+      - name: Compile Python sources
+        run: >-
+          python -m compileall -q
+          custom-connector/src
+          custom-connector/tests
+          custom-graphql
+          dynamic_csv_importer
+          api-lineage-cicd
+          ingestion-automation
+          tests
+
+      - name: Lint and check formatting
+        run: |
+          ruff check .
+          ruff format --check .
+
+      - name: Run model, configuration, import, and request tests
+        run: pytest
+
+  node:
+    name: Node fixture
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: custom-graphql/server
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v7
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: 24
+          cache: npm
+          cache-dependency-path: custom-graphql/server/package-lock.json
+
+      - name: Install locked dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Audit dependencies
+        run: npm audit --audit-level=high
+
+      - name: Check JavaScript syntax
+        run: |
+          node --check index.js
+          node --check resolvers.js
+          node --check schema.js
+
+  compose:
+    name: Docker Compose configuration
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v7
+
+      - name: Download the official OpenMetadata 2.0 RC1 Compose file
+        run: >-
+          curl --fail --silent --show-error --location
+          --output "${RUNNER_TEMP}/docker-compose-postgres.yml"
+          https://github.com/open-metadata/OpenMetadata/releases/download/2.0.0-rc1-release/docker-compose-postgres.yml
+
+      - name: Validate the Compose overlay
+        run: >-
+          docker compose
+          --project-directory "${GITHUB_WORKSPACE}"
+          --file "${RUNNER_TEMP}/docker-compose-postgres.yml"
+          --file custom-connector/docker/compose.override.yml
+          config --quiet
+
+  links:
+    name: Markdown links
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v7
+
+      - name: Check Markdown links
+        uses: lycheeverse/lychee-action@v2.9.0
+        with:
+          args: --no-progress --config .lychee.toml './**/*.md'
+          fail: true
+
+  secrets:
+    name: Gitleaks
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v7
+
+      - name: Scan the repository
+        run: >-
+          docker run --rm
+          --volume "${GITHUB_WORKSPACE}:/repo"
+          zricethezav/gitleaks:v8.27.0
+          dir /repo --redact --no-banner

.lychee.toml

@@ -0,0 +1,12 @@
+accept = [200, 204, 206, 429]
+exclude_all_private = true
+max_retries = 3
+retry_wait_time = 2
+timeout = 20
+exclude = [
+  "^https?://localhost(?::[0-9]+)?(?:/.*)?$",
+  "^https?://127\\.0\\.0\\.1(?::[0-9]+)?(?:/.*)?$",
+  "^https://metadata\\.example\\.com/",
+  "^https://graphql\\.example\\.com/",
+  "^https://raw\\.githubusercontent\\.com/open-metadata/openmetadata-demo/main/mcp/resources/postgres-script\\.sql$",
+]

api-lineage-cicd/tests/test_api_lineage.py

@@ -13,6 +13,7 @@
     configure_from_environment,
     create_lineage,
     load_yaml,
+    verify_lineage,
 )
 
 
@@ -54,21 +55,21 @@ def test_api_yaml_builds_valid_2_0_schema_models() -> None:
 
 def test_api_yaml_builds_valid_2_0_create_requests() -> None:
     """Declarative service, collection, and endpoint inputs must all validate."""
-    orders = load_yaml("orders_api.yaml")
-    collection = orders["collections"][0]
-    endpoint = collection["endpoints"][0]
-
-    service_request = build_api_service_request(orders)
-    collection_request = build_api_collection_request(collection, "orders-api")
-    endpoint_request = build_api_endpoint_request(
-        endpoint,
-        "orders-api.orders",
-    )
-
-    assert service_request.name.root == "orders-api"
-    assert collection_request.service.root == "orders-api"
-    assert endpoint_request.apiCollection.root == "orders-api.orders"
-    assert endpoint_request.requestMethod.value == "POST"
+    request_counts = {}
+    for filename in ("orders_api.yaml", "fulfillment_api.yaml"):
+        raw = load_yaml(filename)
+        service_request = build_api_service_request(raw)
+        requests = [service_request]
+        for collection in raw["collections"]:
+            collection_fqn = f"{raw['serviceName']}.{collection['name']}"
+            requests.append(build_api_collection_request(collection, raw["serviceName"]))
+            requests.extend(
+                build_api_endpoint_request(endpoint, collection_fqn)
+                for endpoint in collection["endpoints"]
+            )
+        request_counts[filename] = len(requests)
+
+    assert request_counts == {"orders_api.yaml": 4, "fulfillment_api.yaml": 4}
 
 
 def test_lineage_yaml_builds_column_lineage_models() -> None:
@@ -92,9 +93,31 @@ def test_lineage_registration_builds_requests_behind_mocked_sdk_boundaries(
     monkeypatch.setattr(Lineage, "add_lineage_request", created.append)
     raw = load_yaml("lineage.yaml")
 
-    create_lineage({"edges": [raw["edges"][0]]})
+    create_lineage(raw)
 
-    assert len(created) == 1
+    assert len(created) == 2
     assert created[0].edge.fromEntity.type == "apiEndpoint"
     assert created[0].edge.toEntity.type == "apiEndpoint"
     assert len(created[0].edge.lineageDetails.columnsLineage) == 2
+    assert len(created[1].edge.lineageDetails.columnsLineage) == 2
+
+
+def test_lineage_verification_uses_mocked_sdk_boundaries(monkeypatch, capsys) -> None:
+    """Verification should request both directions without contacting a server."""
+    endpoint = SimpleNamespace(id="11111111-1111-1111-1111-111111111111")
+    lineage = SimpleNamespace(upstreamEdges=[object()], downstreamEdges=[])
+    calls = []
+    monkeypatch.setattr(APIEndpoints, "retrieve_by_name", lambda _fqn: endpoint)
+
+    def fake_get_entity_lineage(**kwargs):
+        calls.append(kwargs)
+        return lineage
+
+    monkeypatch.setattr(Lineage, "get_entity_lineage", fake_get_entity_lineage)
+
+    verify_lineage()
+
+    assert calls[0]["entity_id"] == endpoint.id
+    assert calls[0]["upstream_depth"] == 2
+    assert calls[0]["downstream_depth"] == 2
+    assert "upstream edges: 1, downstream edges: 0" in capsys.readouterr().out

custom-graphql/tests/test_user_updater.py

@@ -2,14 +2,15 @@
 
 from pathlib import Path
 
-from connector.user_updater import UserUpdaterSource
 from metadata.config.common import load_config_file
 from metadata.generated.schema.entity.domains.domain import Domain, DomainType
 from metadata.generated.schema.entity.teams.user import User
 from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
 
+from connector.user_updater import UserUpdaterSource
+
 
 class FakeMetadata:
     """Small metadata boundary used by the static connector tests."""
@@ -56,6 +57,45 @@ def test_source_uses_the_configured_graphql_endpoint(monkeypatch) -> None:
     assert source.graphql_endpoint == "https://graphql.example.com/query"
 
 
+def test_source_fetches_graphql_records_through_a_mocked_http_boundary(
+    monkeypatch,
+) -> None:
+    """GraphQL response handling should be testable without a fixture server."""
+    calls = []
+
+    class FakeResponse:
+        def raise_for_status(self) -> None:
+            return None
+
+        def json(self) -> dict:
+            return {
+                "data": {
+                    "users": [
+                        {
+                            "email": "analyst@example.com",
+                            "displayName": "Data Analyst",
+                            "domain": "Finance",
+                        }
+                    ]
+                }
+            }
+
+    def fake_post(url: str, **kwargs):
+        calls.append((url, kwargs))
+        return FakeResponse()
+
+    monkeypatch.setattr("connector.user_updater.requests.post", fake_post)
+    monkeypatch.setenv("GRAPHQL_ENDPOINT", "https://graphql.example.com/query")
+    workflow = load_config_file(Path("custom-graphql/user_updater.yaml"))
+    source = UserUpdaterSource.create(workflow["source"], FakeMetadata())
+
+    users = source._fetch_users_from_graphql()
+
+    assert users[0]["email"] == "analyst@example.com"
+    assert calls[0][0] == "https://graphql.example.com/query"
+    assert calls[0][1]["timeout"] == 30
+
+
 def test_source_builds_an_update_for_a_changed_existing_user(monkeypatch) -> None:
     """GraphQL values should become a valid 2.0 CreateUserRequest."""
     monkeypatch.setenv("OPENMETADATA_HOST", "http://localhost:8585/api")