Skip to content

Commit 3b5655b

Browse files
AndrewThienAndrewThien
authored
Optimise Dockerfile of Airflow (#1105)
1 parent 596b891 commit 3b5655b

4 files changed

Lines changed: 69 additions & 95 deletions

File tree

‎.github/workflows/deploy-Airflow.non-prod.yml‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Build, Publish and Deploy Airflow app to Dev
1+
name: Build, Publish and Deploy Airflow app to Non-Prod environments
22

33
on:
44
push:
@@ -52,7 +52,8 @@ jobs:
5252
uses: docker/build-push-action@v5.3.0
5353
with:
5454
context: ./app/airflow
55-
file: ./app/airflow/Dockerfile.webserver
55+
file: ./app/airflow/Dockerfile
56+
build-args: AIRFLOW_COMPONENT=webserver
5657
push: true
5758
platforms: linux/amd64,linux/arm64
5859
tags: ${{ steps.webserver_meta.outputs.tags }}
@@ -73,7 +74,8 @@ jobs:
7374
uses: docker/build-push-action@v5.3.0
7475
with:
7576
context: ./app/airflow
76-
file: ./app/airflow/Dockerfile.scheduler
77+
file: ./app/airflow/Dockerfile
78+
build-args: AIRFLOW_COMPONENT=scheduler
7779
push: true
7880
platforms: linux/amd64,linux/arm64
7981
tags: ${{ steps.scheduler_meta.outputs.tags }}

app/airflow/Dockerfile.scheduler renamed to app/airflow/Dockerfile

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
# TODO: Airflow has version 3.0.0 but this is a breaking change
22
FROM apache/airflow:2.10.5
33

4-
# Install provider packages
4+
# Build argument to determine which entrypoint to use
5+
ARG AIRFLOW_COMPONENT=webserver
6+
7+
# Install provider packages
8+
# (tried using requirements.txt but it took too long to build and install using pyproject.toml didn't work)
59
RUN pip install --no-cache-dir \
610
apache-airflow-providers-microsoft-azure \
711
apache-airflow-providers-postgres \
@@ -15,21 +19,19 @@ RUN pip install --no-cache-dir \
1519
# Copy DAGs to the container
1620
COPY --chown=airflow:root ./dags /opt/airflow/dags
1721

18-
# Create directories with correct ownership from the start
22+
# Create directories with correct ownership
1923
USER root
2024
RUN mkdir -p /opt/airflow/logs /opt/airflow/config && \
2125
chown -R airflow:root /opt/airflow && \
2226
chmod -R g+rw /opt/airflow
2327

24-
# Create entrypoint script
25-
COPY --chown=airflow:root entrypoint-scheduler.sh /entrypoint-scheduler.sh
26-
RUN chmod +x /entrypoint-scheduler.sh
28+
# Copy and setup entrypoint script based on component
29+
COPY --chown=airflow:root entrypoint-${AIRFLOW_COMPONENT}.sh /entrypoint.sh
30+
RUN chmod +x /entrypoint.sh
2731

2832
USER airflow
2933

3034
# Set umask to ensure new files are group writable
3135
ENV UMASK=0002
3236

33-
ENTRYPOINT ["/entrypoint-scheduler.sh"]
34-
35-
37+
ENTRYPOINT ["/entrypoint.sh"]

‎app/airflow/Dockerfile.webserver‎

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎docker-compose.yml‎

Lines changed: 54 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,38 @@
11
name: carrot-mapper-dev
22

3+
x-airflow-common:
4+
&airflow-common
5+
profiles: ["main"]
6+
volumes:
7+
- ./app/airflow/dags:/opt/airflow/dags
8+
depends_on:
9+
db:
10+
condition: service_started
11+
minio:
12+
condition: service_started
13+
api:
14+
condition: service_healthy
15+
environment:
16+
&airflow-common-env
17+
AIRFLOW__CORE__EXECUTOR: LocalExecutor
18+
# Database connection here needs to be in URI format. With real values, which have special characters, they need to be encoded using the ref. in https://www.w3schools.com/tags/ref_urlencode.ASP.
19+
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://postgres:postgres@db:5432/postgres?options=-csearch_path%3Dairflow
20+
AIRFLOW__DATABASE__SQL_ALCHEMY_SCHEMA: airflow
21+
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
22+
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
23+
AIRFLOW__WEBSERVER__WEB_SERVER_PORT: 8080
24+
AIRFLOW__API__AUTH_BACKEND: airflow.api.auth.backend.basic_auth
25+
# This connection can be in separate variables (then don't need to do URL encoding), but it can reuse the same connection string with SQL_ALCHEMY_CONN (without extras)
26+
AIRFLOW_CONN_POSTGRES_DB_CONN: postgresql+psycopg2://postgres:postgres@db:5432/postgres
27+
# The blob storage where Airflow get/put necessary files
28+
STORAGE_TYPE: minio
29+
# Minio connection variables (don't need to do URL encoding here)
30+
AIRFLOW_VAR_MINIO_ENDPOINT: http://minio:9000
31+
AIRFLOW_VAR_MINIO_ACCESS_KEY: minioadmin
32+
AIRFLOW_VAR_MINIO_SECRET_KEY: minioadmin
33+
# WASB connection string (don't need to do URL encoding here)
34+
AIRFLOW_VAR_WASB_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1
35+
336
services:
437
db:
538
image: postgres:14
@@ -51,6 +84,8 @@ services:
5184
depends_on:
5285
api:
5386
condition: service_healthy
87+
airflow-webserver:
88+
condition: service_healthy
5489

5590
api:
5691
image: carrot-backend
@@ -189,70 +224,37 @@ services:
189224
retries: 5
190225

191226
airflow-webserver:
192-
profiles: ["main"]
227+
<<: *airflow-common
193228
build:
194229
context: app/airflow
195-
dockerfile: Dockerfile.webserver
196-
depends_on:
197-
db:
198-
condition: service_started
199-
api:
200-
condition: service_healthy
230+
dockerfile: Dockerfile
231+
args:
232+
AIRFLOW_COMPONENT: webserver
233+
ports:
234+
- "8080:8080"
235+
healthcheck:
236+
test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
237+
interval: 10s
238+
timeout: 5s
239+
retries: 5
201240
environment:
202-
AIRFLOW__CORE__EXECUTOR: LocalExecutor
203-
# Database connection here needs to be in URI format. With real values, which have special characters, they need to be encoded using the ref. in https://www.w3schools.com/tags/ref_urlencode.ASP.
204-
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://postgres:postgres@db:5432/postgres?options=-csearch_path%3Dairflow
205-
AIRFLOW__DATABASE__SQL_ALCHEMY_SCHEMA: airflow
206-
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
207-
AIRFLOW__WEBSERVER__WEB_SERVER_PORT: 8080
208-
AIRFLOW__WEBSERVER__SECRET_KEY: "test"
209-
AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
210-
AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "False"
211-
# This connection can be in separate variables (then don't need to do URL encoding), but it can reuse the same connection string with SQL_ALCHEMY_CONN (without extras)
212-
AIRFLOW_CONN_POSTGRES_DB_CONN: postgresql+psycopg2://postgres:postgres@db:5432/postgres
213-
# The blob storage where Airflow get/put necessary files
214-
STORAGE_TYPE: minio
215-
# Minio connection variables (don't need to do URL encoding here)
216-
AIRFLOW_VAR_MINIO_ENDPOINT: http://minio:9000
217-
AIRFLOW_VAR_MINIO_ACCESS_KEY: minioadmin
218-
AIRFLOW_VAR_MINIO_SECRET_KEY: minioadmin
219-
# WASB connection string (don't need to do URL encoding here)
220-
AIRFLOW_VAR_WASB_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1
241+
<<: *airflow-common-env
221242
# Admin user configuration to access the Airlfow API
243+
# TODO: add optional admin user configuration in the docs
222244
AIRFLOW_ADMIN_USERNAME: admin
223245
AIRFLOW_ADMIN_PASSWORD: admin
224-
# TODO: add optional admin user configuration in the docs
225-
volumes:
226-
- ./app/airflow/dags:/opt/airflow/dags
227-
ports:
228-
- "8080:8080"
229-
246+
230247
scheduler:
231-
profiles: ["main"]
248+
<<: *airflow-common
232249
build:
233250
context: app/airflow
234-
dockerfile: Dockerfile.scheduler
235-
depends_on:
236-
api:
237-
condition: service_healthy
238-
airflow-webserver:
239-
condition: service_started
251+
dockerfile: Dockerfile
252+
args:
253+
AIRFLOW_COMPONENT: scheduler
240254
environment:
241-
AIRFLOW__CORE__EXECUTOR: LocalExecutor
255+
<<: *airflow-common-env
242256
AIRFLOW__CORE__EXECUTE_TASKS_NEW_PYTHON_INTERPRETER: True
243-
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://postgres:postgres@db:5432/postgres?options=-csearch_path%3Dairflow
244-
AIRFLOW__DATABASE__SQL_ALCHEMY_SCHEMA: airflow
245-
AIRFLOW__WEBSERVER__SECRET_KEY: "test"
246-
AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth
247-
AIRFLOW_CONN_POSTGRES_DB_CONN: postgresql+psycopg2://postgres:postgres@db:5432/postgres
248-
STORAGE_TYPE: minio
249-
AIRFLOW_VAR_MINIO_ENDPOINT: http://minio:9000
250-
AIRFLOW_VAR_MINIO_ACCESS_KEY: minioadmin
251-
AIRFLOW_VAR_MINIO_SECRET_KEY: minioadmin
252-
AIRFLOW_VAR_WASB_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;QueueEndpoint=http://azurite:10001/devstoreaccount1;TableEndpoint=http://azurite:10002/devstoreaccount1
253257
AIRFLOW_DEBUG_MODE: True
254-
volumes:
255-
- ./app/airflow/dags:/opt/airflow/dags
256258

257259
volumes:
258260
db_data:

0 commit comments

Comments
 (0)