Skip to content

Commit 30b427f

Browse files
abhishekagrawal-atlangithub-advanced-security[bot]sachi-atlan
authored
chore: adding e2e to sample apps (#111)
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> Co-authored-by: Sachi Patankar <[email protected]>
1 parent bd47674 commit 30b427f

File tree

69 files changed

+2680
-17
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+2680
-17
lines changed

‎.github/workflows/e2e-test.yaml‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
name: E2E Application Test
2+
permissions:
3+
contents: read
4+
pull-requests: write
5+
on:
6+
push:
7+
branches:
8+
- main
9+
- e2e
10+
pull_request:
11+
types: [labeled, reopened, synchronize, opened]
12+
13+
jobs:
14+
test:
15+
runs-on: ${{ matrix.os }}
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os }}-${{ matrix.app.name }}
18+
cancel-in-progress: false
19+
timeout-minutes: 30
20+
if: ${{ github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'e2e-test')) }}
21+
strategy:
22+
fail-fast: false
23+
max-parallel: 3
24+
matrix:
25+
os: [ubuntu-latest, windows-latest]
26+
app:
27+
- name: hello_world
28+
path: quickstart/hello_world
29+
application_name: hello-world
30+
- name: polyglot
31+
path: quickstart/polyglot
32+
application_name: polyglot
33+
- name: mysql
34+
path: connectors/mysql
35+
application_name: mysql
36+
- name: workflows_observability
37+
path: utilities/workflows_observability
38+
application_name: workflows-observability
39+
- name: giphy
40+
path: quickstart/giphy
41+
application_name: giphy
42+
# - name: ai_giphy
43+
# path: quickstart/ai_giphy
44+
# application_name: ai-giphy
45+
46+
steps:
47+
- uses: actions/[email protected]
48+
49+
# Install Dapr
50+
- name: Install Dapr CLI (Linux/macOS)
51+
if: runner.os != 'Windows'
52+
shell: bash
53+
run: |
54+
wget -q https://raw.githubusercontent.com/dapr/cli/master/install/install.sh -O - | /bin/bash -s 1.16.2
55+
dapr init --runtime-version 1.16.0 --slim
56+
57+
- name: Install Dapr CLI (Windows)
58+
if: runner.os == 'Windows'
59+
shell: pwsh
60+
run: |
61+
$script=iwr -useb https://raw.githubusercontent.com/dapr/cli/master/install/install.ps1; $block=[ScriptBlock]::Create($script); invoke-command -ScriptBlock $block -ArgumentList 1.16.2, "$env:USERPROFILE\.dapr\bin\"
62+
$env:Path += ";$env:USERPROFILE\.dapr\bin\"
63+
dapr init --runtime-version 1.16.0 --slim
64+
65+
# Install Temporal
66+
- name: Install Temporal CLI and Start Server (Linux/macOS)
67+
if: runner.os != 'Windows'
68+
shell: bash
69+
run: |
70+
curl -sSf https://temporal.download/cli.sh | sh
71+
export PATH="$HOME/.temporalio/bin:$PATH"
72+
temporal server start-dev --db-filename /tmp/temporal-${{ matrix.app.name }}.db &
73+
sleep 5
74+
75+
- name: Install Temporal CLI and Start Server (Windows)
76+
if: runner.os == 'Windows'
77+
shell: pwsh
78+
run: |
79+
# Create a directory for Temporal CLI
80+
New-Item -ItemType Directory -Force -Path "$env:USERPROFILE\.temporalio\bin"
81+
# Download Temporal CLI
82+
Invoke-WebRequest -Uri "https://temporal.download/cli/archive/latest?platform=windows&arch=amd64" -OutFile "$env:USERPROFILE\.temporalio\temporal.zip"
83+
# Extract and install
84+
Expand-Archive -Path "$env:USERPROFILE\.temporalio\temporal.zip" -DestinationPath "$env:USERPROFILE\.temporalio\bin" -Force
85+
# Add to PATH
86+
$env:Path += ";$env:USERPROFILE\.temporalio\bin"
87+
# Start Temporal server
88+
$dbPath = "$env:TEMP\temporal-${{ matrix.app.name }}.db"
89+
Start-Process -FilePath "temporal" -ArgumentList "server", "start-dev", "--db-filename", $dbPath -NoNewWindow
90+
Start-Sleep -Seconds 5
91+
92+
# Setup Python and uv
93+
- name: Setup Python
94+
uses: actions/setup-python@v5
95+
with:
96+
python-version: '3.11'
97+
98+
- name: Setup uv
99+
uses: astral-sh/setup-uv@v5
100+
with:
101+
version: "0.7.3"
102+
103+
# Set up PATH for DAPR and Temporal globally
104+
- name: Setup PATH for DAPR and Temporal (Linux/macOS)
105+
if: runner.os != 'Windows'
106+
shell: bash
107+
run: |
108+
echo "$HOME/.dapr/bin" >> $GITHUB_PATH
109+
echo "$HOME/.temporalio/bin" >> $GITHUB_PATH
110+
111+
- name: Setup PATH for DAPR and Temporal (Windows)
112+
if: runner.os == 'Windows'
113+
shell: pwsh
114+
run: |
115+
echo "$env:USERPROFILE\.dapr\bin" >> $env:GITHUB_PATH
116+
echo "$env:USERPROFILE\.temporalio\bin" >> $env:GITHUB_PATH
117+
118+
# Sync dependencies in app directory
119+
- name: Sync dependencies
120+
working-directory: ${{ matrix.app.path }}
121+
shell: bash
122+
run: |
123+
uv sync --all-extras --all-groups
124+
125+
# Build Java JAR for polyglot (if needed)
126+
- name: Build Java JAR for polyglot
127+
if: matrix.app.name == 'polyglot'
128+
working-directory: ${{ matrix.app.path }}
129+
shell: bash
130+
run: |
131+
uv run poe build-java || echo "Java build not required or failed"
132+
133+
# Download components and start dapr and temporal services
134+
- name: Start Platform Services
135+
working-directory: ${{ matrix.app.path }}
136+
shell: bash
137+
run: |
138+
uv run poe download-components
139+
uv run poe start-deps
140+
# Wait for Dapr sidecar to be ready
141+
sleep 5
142+
143+
# Start the application
144+
- name: Start the application
145+
id: start_app
146+
working-directory: ${{ matrix.app.path }}
147+
shell: bash
148+
env:
149+
ATLAN_LOCAL_DEVELOPMENT: true
150+
ATLAN_APPLICATION_NAME: ${{ matrix.app.application_name }}
151+
ATLAN_API_KEY: ${{ secrets.ATLAN_API_KEY }}
152+
ATLAN_BASE_URL: ${{ secrets.ATLAN_BASE_URL }}
153+
GIPHY_API_KEY: ${{ secrets.GIPHY_API_KEY }}
154+
SMTP_HOST: ${{ secrets.SMTP_HOST }}
155+
SMTP_PORT: ${{ secrets.SMTP_PORT }}
156+
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
157+
SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }}
158+
SMTP_SENDER: ${{ secrets.SMTP_SENDER }}
159+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
160+
OPENAI_MODEL_NAME: ${{ secrets.OPENAI_MODEL_NAME }}
161+
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
162+
run: |
163+
uv run python main.py &
164+
sleep 20
165+
166+
- name: Run the end-to-end integration tests
167+
working-directory: ${{ matrix.app.path }}
168+
shell: bash
169+
env:
170+
ATLAN_LOCAL_DEVELOPMENT: true
171+
ATLAN_APPLICATION_NAME: ${{ matrix.app.application_name }}
172+
ATLAN_API_KEY: ${{ secrets.ATLAN_API_KEY }}
173+
ATLAN_BASE_URL: ${{ secrets.ATLAN_BASE_URL }}
174+
E2E_MYSQL_BASIC_AUTH_HOST: ${{ secrets.E2E_MYSQL_BASIC_AUTH_HOST }}
175+
E2E_MYSQL_BASIC_AUTH_PASSWORD: ${{ secrets.E2E_MYSQL_BASIC_AUTH_PASSWORD }}
176+
E2E_MYSQL_BASIC_AUTH_PORT: ${{ secrets.E2E_MYSQL_BASIC_AUTH_PORT }}
177+
E2E_MYSQL_BASIC_AUTH_USERNAME: ${{ secrets.E2E_MYSQL_BASIC_AUTH_USERNAME }}
178+
E2E_MYSQL_BASIC_DATABASE: ${{ secrets.E2E_MYSQL_BASIC_DATABASE }}
179+
GIPHY_API_KEY: ${{ secrets.GIPHY_API_KEY }}
180+
SMTP_HOST: ${{ secrets.SMTP_HOST }}
181+
SMTP_PORT: ${{ secrets.SMTP_PORT }}
182+
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
183+
SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }}
184+
SMTP_SENDER: ${{ secrets.SMTP_SENDER }}
185+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
186+
OPENAI_MODEL_NAME: ${{ secrets.OPENAI_MODEL_NAME }}
187+
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
188+
run: |
189+
uv run coverage run -m pytest tests/e2e --capture=no --log-cli-level=INFO -v --full-trace
190+
191+
# Stop all services
192+
- name: Stop all services (Linux/macOS)
193+
if: always() && runner.os != 'Windows'
194+
working-directory: ${{ matrix.app.path }}
195+
shell: bash
196+
run: |
197+
export PATH="$HOME/.temporalio/bin:$PATH"
198+
kill $(lsof -t -i :8000) || true
199+
uv run poe stop-deps || true
200+
201+
- name: Stop all services (Windows)
202+
if: always() && runner.os == 'Windows'
203+
working-directory: ${{ matrix.app.path }}
204+
shell: pwsh
205+
run: |
206+
$env:Path += ";$env:USERPROFILE\.temporalio\bin"
207+
# Stop process on port 8000
208+
$process = Get-NetTCPConnection -LocalPort 8000 -ErrorAction SilentlyContinue | Select-Object -ExpandProperty OwningProcess -Unique
209+
if ($process) { Stop-Process -Id $process -Force -ErrorAction SilentlyContinue }
210+
uv run poe stop-deps || true

‎.github/workflows/pull-request.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,12 +144,12 @@ jobs:
144144
- name: Run tests 🧪
145145
working-directory: ${{ matrix.app }}
146146
run: |
147-
uv run python -m pytest tests/ -v --tb=short
147+
uv run python -m pytest tests/unit/ -v --tb=short
148148
149149
- name: Run tests with coverage 🔄
150150
working-directory: ${{ matrix.app }}
151151
run: |
152-
uv run coverage run -m pytest tests/ -v
152+
uv run coverage run -m pytest tests/unit/ -v
153153
uv run coverage report --show-missing
154154
uv run coverage xml
155155

‎connectors/anaplan/app/workflows.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from temporalio.common import RetryPolicy
99

1010
# Constants for activity retry policies
11-
ACTIVITY_RETRY_MAX_ATTEMPTS = 3
11+
ACTIVITY_RETRY_MAX_ATTEMPTS = 6 # 1 initial attempt + 5 retries
1212
ACTIVITY_RETRY_BACKOFF_COEFFICIENT = 2
1313

1414
logger = get_logger(__name__)

‎connectors/anaplan/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ test = [
3131

3232
[tool.poe.tasks]
3333
# Dapr and Temporal service tasks
34-
start-dapr = "dapr run --enable-api-logging --log-level debug --app-id app --app-port 3000 --dapr-http-port 3500 --dapr-grpc-port 50001 --dapr-http-max-request-size 1024 --resources-path components"
34+
start-dapr = "dapr run --enable-api-logging --log-level debug --app-id app --app-port 3000 --dapr-http-port 3500 --dapr-grpc-port 50001 --max-body-size 1024Mi --resources-path components"
3535
start-temporal = "temporal server start-dev --db-filename ./temporal.db"
3636
start-deps.shell = "poe start-dapr & poe start-temporal &"
3737
stop-deps.shell = "lsof -ti:3000,3500,7233,50001 | xargs kill -9 2>/dev/null || true"

‎connectors/mysql/tests/e2e/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""E2E tests for mysql application."""

‎connectors/mysql/tests/e2e/test_mysql_workflow/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""E2E tests for mysql workflow."""

‎connectors/mysql/tests/e2e/test_mysql_workflow/config.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
test_workflow_args:
2+
credentials:
3+
user: $E2E_MYSQL_BASIC_AUTH_USERNAME
4+
password: $E2E_MYSQL_BASIC_AUTH_PASSWORD
5+
host: $E2E_MYSQL_BASIC_AUTH_HOST
6+
port: $E2E_MYSQL_BASIC_AUTH_PORT
7+
database: $E2E_MYSQL_BASIC_DATABASE
8+
authType: "basic"
9+
metadata:
10+
exclude-filter: "{}"
11+
include-filter: "{}"
12+
temp-table-regex: ""
13+
extraction-method: "direct"
14+
connection:
15+
connection_name: "mysql-test"
16+
connection_qualified_name: "default/mysql/test"
17+
test_name: "test_mysql_workflow"
18+
server_config:
19+
server_host: "http://localhost:8000"
20+
server_version: "workflows/v1"
21+
expected_api_responses:
22+
auth:
23+
success: true
24+
message: "Authentication successful"
25+
metadata:
26+
success: true
27+
preflight_check:
28+
success: true
29+
data:
30+
databaseSchemaCheck:
31+
success: true
32+
successMessage: "Schemas and Databases check successful"
33+
failureMessage: ""
34+
tablesCheck:
35+
success: true
36+
successMessage: "Tables check successful"
37+
failureMessage: ""
38+

‎connectors/mysql/tests/e2e/test_mysql_workflow/schema/raw/column.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
schema_type: dataframe
2+
version: 0.22.1
3+
columns:
4+
table_catalog:
5+
title: Table Catalog Name Check
6+
description: Check if the catalog name of records in the raw data is matched as expected
7+
dtype: str
8+
nullable: false
9+
unique: false
10+
coerce: false
11+
required: true
12+
regex: false
13+
table_schema:
14+
title: Table Schema Name Check
15+
description: Check if the schema name of records in the raw data is matched as expected
16+
dtype: str
17+
nullable: false
18+
unique: false
19+
coerce: false
20+
required: true
21+
regex: false
22+
table_name:
23+
title: Check Table Names
24+
description: Check if the table names are within the expected list
25+
dtype: str
26+
nullable: false
27+
unique: false
28+
coerce: false
29+
required: true
30+
regex: false
31+
column_name:
32+
title: Check Column Names
33+
description: Check if the column names are present
34+
dtype: str
35+
nullable: false
36+
unique: false
37+
coerce: false
38+
required: true
39+
regex: false
40+
ordinal_position:
41+
title: Ordinal Position
42+
description: The ordinal position of the column
43+
dtype: Int64
44+
nullable: false
45+
unique: false
46+
coerce: true
47+
required: true
48+
regex: false
49+
is_nullable:
50+
title: Is Nullable
51+
description: Whether the column allows null values
52+
dtype: str
53+
nullable: false
54+
unique: false
55+
coerce: false
56+
required: true
57+
regex: false
58+
data_type:
59+
title: Data Type
60+
description: The data type of the column
61+
dtype: str
62+
nullable: false
63+
unique: false
64+
coerce: false
65+
required: true
66+
regex: false
67+
is_autoincrement:
68+
title: Is Auto Increment
69+
description: Whether the column is auto-increment
70+
dtype: str
71+
nullable: false
72+
unique: false
73+
coerce: false
74+
required: true
75+
regex: false
76+
checks:
77+
check_record_count_ge:
78+
value: 1
79+
options:
80+
raise_warning: false
81+
82+
index: null
83+
dtype: null
84+
coerce: false
85+
strict: false
86+
name: null
87+
ordered: false
88+
unique: null
89+
report_duplicates: all
90+
unique_column_names: true
91+
add_missing_columns: false
92+
title: Raw Column Pandera Schema
93+
description: Pandera schema definition for the raw column output
94+

0 commit comments

Comments
 (0)