Skip to content

Commit 6920da3

Browse files
ayush-shahayush-shah
authored
Merge pull request #73 from open-metadata/ayush-shah-codex/python-sdk-retrieval-fix
feat: revamp OpenMetadata 2.0 RC1 demo catalog
2 parents 63d9132 + 30d5558 commit 6920da3

186 files changed

Lines changed: 7053 additions & 17063 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/CODEOWNERS‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
/README.md @open-metadata/ingestion @open-metadata/documentation-team
2+
/.github/ @open-metadata/ingestion @open-metadata/documentation-team
3+
/custom-connector/ @open-metadata/ingestion @open-metadata/documentation-team
4+
/custom-graphql/ @open-metadata/ingestion @open-metadata/documentation-team
5+
/dynamic_csv_importer/ @open-metadata/ingestion @open-metadata/documentation-team
6+
/api-lineage-cicd/ @open-metadata/ingestion @open-metadata/documentation-team
7+
/ingestion-automation/ @open-metadata/ingestion @open-metadata/documentation-team
8+
/mcp/ @open-metadata/documentation-team

‎.github/dependabot.yml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: pip
4+
directory: /
5+
schedule:
6+
interval: weekly
7+
day: monday
8+
time: "06:00"
9+
timezone: Etc/UTC
10+
groups:
11+
sdk-facade-python:
12+
patterns: ["*"]
13+
14+
- package-ecosystem: pip
15+
directory: /custom-connector
16+
schedule:
17+
interval: weekly
18+
day: monday
19+
time: "06:00"
20+
timezone: Etc/UTC
21+
groups:
22+
custom-connector-python:
23+
patterns: ["*"]
24+
25+
- package-ecosystem: pip
26+
directory: /custom-graphql
27+
schedule:
28+
interval: weekly
29+
day: monday
30+
time: "06:00"
31+
timezone: Etc/UTC
32+
groups:
33+
custom-graphql-python:
34+
patterns: ["*"]
35+
36+
- package-ecosystem: pip
37+
directory: /dynamic_csv_importer
38+
schedule:
39+
interval: weekly
40+
day: monday
41+
time: "06:00"
42+
timezone: Etc/UTC
43+
groups:
44+
dynamic-csv-python:
45+
patterns: ["*"]
46+
47+
- package-ecosystem: pip
48+
directory: /api-lineage-cicd
49+
schedule:
50+
interval: weekly
51+
day: monday
52+
time: "06:00"
53+
timezone: Etc/UTC
54+
groups:
55+
api-lineage-python:
56+
patterns: ["*"]
57+
58+
- package-ecosystem: pip
59+
directory: /ingestion-automation
60+
schedule:
61+
interval: weekly
62+
day: monday
63+
time: "06:00"
64+
timezone: Etc/UTC
65+
groups:
66+
ingestion-automation-python:
67+
patterns: ["*"]
68+
69+
- package-ecosystem: npm
70+
directory: /custom-graphql/server
71+
schedule:
72+
interval: weekly
73+
day: monday
74+
time: "06:00"
75+
timezone: Etc/UTC
76+
groups:
77+
graphql-server:
78+
patterns: ["*"]
79+
80+
- package-ecosystem: docker
81+
directory: /custom-connector/docker
82+
schedule:
83+
interval: weekly
84+
day: monday
85+
time: "06:00"
86+
timezone: Etc/UTC
87+
88+
- package-ecosystem: github-actions
89+
directory: /
90+
schedule:
91+
interval: weekly
92+
day: monday
93+
time: "06:00"
94+
timezone: Etc/UTC
95+
groups:
96+
github-actions:
97+
patterns: ["*"]

‎.github/workflows/snowflake-ingestion.yaml‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎.github/workflows/validate-examples.yml‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
name: Validate maintained examples
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
workflow_dispatch:
9+
schedule:
10+
- cron: "0 6 * * 1"
11+
12+
permissions:
13+
contents: read
14+
15+
jobs:
16+
python:
17+
name: Python ${{ matrix.python-version }}
18+
runs-on: ubuntu-latest
19+
strategy:
20+
fail-fast: false
21+
matrix:
22+
python-version: ["3.10", "3.11", "3.12"]
23+
steps:
24+
- name: Check out the repository
25+
uses: actions/checkout@v7
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@v7
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
cache: pip
32+
cache-dependency-path: |
33+
requirements-sdk.txt
34+
custom-connector/pyproject.toml
35+
custom-graphql/pyproject.toml
36+
dynamic_csv_importer/requirements.txt
37+
api-lineage-cicd/requirements.txt
38+
ingestion-automation/requirements.txt
39+
40+
- name: Install exact OpenMetadata RC and test dependencies
41+
run: |
42+
python -m pip install --upgrade pip
43+
python -m pip install --requirement requirements-sdk.txt
44+
python -m pip install --editable './custom-connector[test]'
45+
python -m pip install --editable ./custom-graphql
46+
python -m pip install \
47+
--requirement dynamic_csv_importer/requirements.txt \
48+
--requirement api-lineage-cicd/requirements.txt \
49+
--requirement ingestion-automation/requirements.txt
50+
51+
- name: Verify the installed OpenMetadata version
52+
run: |
53+
python - <<'PY'
54+
from importlib.metadata import version
55+
56+
assert version("openmetadata-ingestion") == "2.0.0.0rc1"
57+
PY
58+
59+
- name: Compile Python sources
60+
run: >-
61+
python -m compileall -q
62+
custom-connector/src
63+
custom-connector/tests
64+
custom-graphql
65+
dynamic_csv_importer
66+
api-lineage-cicd
67+
ingestion-automation
68+
catalog-assets
69+
analytics-assets
70+
storage-assets
71+
governance-assets
72+
identity-and-settings
73+
data-quality
74+
tests
75+
76+
- name: Lint and check formatting
77+
run: |
78+
ruff check .
79+
ruff format --check .
80+
81+
- name: Run offline RC1 model, facade, import, and request tests
82+
run: pytest
83+
84+
node:
85+
name: Node fixture
86+
runs-on: ubuntu-latest
87+
defaults:
88+
run:
89+
working-directory: custom-graphql/server
90+
steps:
91+
- name: Check out the repository
92+
uses: actions/checkout@v7
93+
94+
- name: Set up Node.js
95+
uses: actions/setup-node@v7
96+
with:
97+
node-version: 24
98+
cache: npm
99+
cache-dependency-path: custom-graphql/server/package-lock.json
100+
101+
- name: Install locked dependencies
102+
run: npm ci --ignore-scripts
103+
104+
- name: Audit dependencies
105+
run: npm audit --audit-level=high
106+
107+
- name: Check JavaScript syntax
108+
run: |
109+
node --check index.js
110+
node --check resolvers.js
111+
node --check schema.js
112+
113+
compose:
114+
name: Docker Compose configuration
115+
runs-on: ubuntu-latest
116+
steps:
117+
- name: Check out the repository
118+
uses: actions/checkout@v7
119+
120+
- name: Download the official OpenMetadata 2.0 RC1 Compose file
121+
run: >-
122+
curl --fail --silent --show-error --location
123+
--output "${RUNNER_TEMP}/docker-compose-postgres.yml"
124+
https://github.com/open-metadata/OpenMetadata/releases/download/2.0.0-rc1-release/docker-compose-postgres.yml
125+
126+
- name: Validate the Compose overlay
127+
run: >-
128+
docker compose
129+
--project-directory "${GITHUB_WORKSPACE}"
130+
--file "${RUNNER_TEMP}/docker-compose-postgres.yml"
131+
--file custom-connector/docker/compose.override.yml
132+
config --quiet
133+
134+
links:
135+
name: Markdown links
136+
runs-on: ubuntu-latest
137+
steps:
138+
- name: Check out the repository
139+
uses: actions/checkout@v7
140+
141+
- name: Check Markdown links
142+
uses: lycheeverse/lychee-action@v2.9.0
143+
with:
144+
args: --no-progress --config .lychee.toml './**/*.md'
145+
fail: true
146+
147+
secrets:
148+
name: Gitleaks
149+
runs-on: ubuntu-latest
150+
steps:
151+
- name: Check out the repository
152+
uses: actions/checkout@v7
153+
154+
- name: Scan the repository
155+
run: >-
156+
docker run --rm
157+
--volume "${GITHUB_WORKSPACE}:/repo"
158+
zricethezav/gitleaks:v8.27.0
159+
dir /repo --redact --no-banner

‎.gitmodules‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎.lychee.toml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
accept = [200, 204, 206, 429]
2+
exclude_all_private = true
3+
max_retries = 3
4+
retry_wait_time = 2
5+
timeout = 20
6+
exclude = [
7+
"^https?://localhost(?::[0-9]+)?(?:/.*)?$",
8+
"^https?://127\\.0\\.0\\.1(?::[0-9]+)?(?:/.*)?$",
9+
"^https://metadata\\.example\\.com/",
10+
"^https://graphql\\.example\\.com/",
11+
]

0 commit comments

Comments
 (0)