Skip to content

Commit 6264636

Browse files
vancauwevancauwe
committed
feat(airflow): docker and DAG
1 parent b8772a5 commit 6264636

4 files changed

Lines changed: 160 additions & 0 deletions

File tree

‎.github/workflows/docker-airflow.yml‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
name: Build and Push Docker Image for Airflow
2+
3+
on:
4+
release:
5+
type: [published]
6+
#trigger on each commit of each branch and push with latest tag
7+
push :
8+
pull_request:
9+
paths:
10+
- 'pyproject.toml'
11+
- './docker/**'
12+
- '.github/workflows/**'
13+
14+
env:
15+
REGISTRY: ghcr.io
16+
17+
jobs:
18+
build-image:
19+
runs-on: ubuntu-latest
20+
#if: github.ref != 'refs/heads/main'
21+
permissions:
22+
contents: read
23+
packages: write
24+
25+
steps:
26+
# https://github.com/actions/checkout
27+
- name: checkout repository
28+
uses: actions/checkout@v4
29+
30+
- name: lowercase image name
31+
run: |
32+
echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}-airflow" >> ${GITHUB_ENV}
33+
34+
# https://github.com/docker/setup-qemu-action
35+
- name: Set up QEMU
36+
uses: docker/setup-qemu-action@v3.0.0
37+
38+
# https://github.com/docker/setup-buildx-action
39+
- name: Set up Docker Buildx
40+
id: buildx
41+
uses: docker/setup-buildx-action@v3.0.0
42+
43+
- name: Get current release version
44+
id: release-version
45+
run: |
46+
version=$(grep -E '^version += +' pyproject.toml | sed -E 's/.*= +//' | sed "s/['\"]//g")
47+
echo "version=${version}" >> $GITHUB_OUTPUT
48+
echo "version_build=${version}_"$(git rev-parse --short "$GITHUB_SHA") >> $GITHUB_OUTPUT
49+
50+
# https://github.com/docker/build-push-action
51+
- name: Build Docker image
52+
uses: docker/build-push-action@v5.0.0
53+
with:
54+
context: .
55+
platforms: linux/amd64
56+
file: tools/images/Dockerfile-airflow
57+
push: false
58+
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.release-version.outputs.version_build }}
59+
build-args: PLATFORM=cu124
60+
outputs: type=image,annotation-index.org.opencontainers.image.description=Run a Graph Neural Network.
61+
62+
push-image:
63+
runs-on: ubuntu-latest
64+
#if: github.ref == 'refs/heads/main'
65+
permissions:
66+
contents: read
67+
packages: write
68+
69+
steps:
70+
# https://github.com/actions/checkout
71+
- name: checkout repository
72+
uses: actions/checkout@v4
73+
74+
- name: lowercase image name
75+
run: |
76+
echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}-airflow" >> ${GITHUB_ENV}
77+
78+
# https://github.com/docker/setup-qemu-action
79+
- name: Set up QEMU
80+
uses: docker/setup-qemu-action@v3.0.0
81+
82+
# https://github.com/docker/setup-buildx-action
83+
- name: Set up Docker Buildx
84+
id: buildx
85+
uses: docker/setup-buildx-action@v3.0.0
86+
87+
- name: Get current release version
88+
id: release-version
89+
run: |
90+
version=$(grep -E '^version += +' pyproject.toml | sed -E 's/.*= +//' | sed "s/['\"]//g")
91+
echo "version=${version}" >> $GITHUB_OUTPUT
92+
echo "version_build=${version}_"$(git rev-parse --short "$GITHUB_SHA") >> $GITHUB_OUTPUT
93+
94+
# https://github.com/docker/login-action
95+
- name: Log in to the Container registry
96+
uses: docker/login-action@v3.0.0
97+
with:
98+
registry: ${{ env.REGISTRY }}
99+
username: ${{ github.actor }}
100+
password: ${{ secrets.GITHUB_TOKEN }}
101+
102+
# https://github.com/docker/metadata-action
103+
- name: Extract metadata (tags, labels) for Docker
104+
id: meta
105+
uses: docker/metadata-action@v5.0.0
106+
with:
107+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
108+
tags: |
109+
type=raw,value=latest,enable=${{ github.event_name == 'push' }}
110+
type=raw,value=${{ needs.build-image.outputs.version }},enable=${{ github.event_name == 'release' }}
111+
# for future:
112+
# type=raw,value=${{ needs.build-image.outputs.version_build }},enable=${{ github.event_name == 'push' }}
113+
114+
# https://github.com/docker/build-push-action
115+
- name: Push Docker image
116+
uses: docker/build-push-action@v5.0.0
117+
with:
118+
context: .
119+
platforms: linux/amd64
120+
file: tools/images/Dockerfile-airflow
121+
push: true
122+
tags: ${{ steps.meta.outputs.tags }}
123+
labels: ${{ steps.meta.outputs.labels }}
124+
build-args: PLATFORM=cu124
125+
outputs: type=image,annotation-index.org.opencontainers.image.description=Run a Graph Neural Network.

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
.env
2+
.DS_Store
23

34
# Ignore all Airflow related configurations
45
**/airflow/config/

‎src/airflow/src/inference.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import requests
2+
import logging
3+
import os
4+
5+
BASE_URL = os.getenv('ML_BASE_URL', 'https://your-api.com')
6+
NEO4J_DB = os.getenv('NEO4J_DATABASE')
7+
API_TOKEN = os.getenv('API_TOKEN')
8+
9+
def call_ml_inference():
10+
url = f"{BASE_URL}/v1/inference/epfl/{NEO4J_DB}"
11+
headers = {
12+
'Authorization': f'Bearer {API_TOKEN}'
13+
}
14+
15+
try:
16+
response = requests.get(url, headers=headers, timeout=30)
17+
response.raise_for_status()
18+
result = response.json()
19+
logging.info("Inference API response: %s", result)
20+
except requests.exceptions.RequestException as e:
21+
logging.error("Failed to call inference endpoint: %s", str(e))
22+
raise
23+
24+
if __name__ == "__main__":
25+
call_ml_inference()

‎tools/images/Dockerfile-airflow‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM python:3.9-slim
2+
3+
WORKDIR /app
4+
5+
RUN pip install --no-cache-dir requests
6+
7+
COPY ../src/inference.py .
8+
9+
ENTRYPOINT ["python", "inference.py"]

0 commit comments

Comments
 (0)