feat(ariflow): docker and dag set-up

Author: vancauwe

.github/workflows/docker-airflow.yml

@@ -8,7 +8,7 @@ on:
   pull_request:
     paths:
       - 'pyproject.toml'
-      - './docker/**'
+      - 'tools/images/**'
       - '.github/workflows/**'
 
 env:

.gitignore

@@ -1,11 +1,12 @@
 .env
 .DS_Store
 
+# Ignore any file anywhere in the repository called env_variables
+src/airflow/src/env_variables.json
+
 # Ignore all Airflow related configurations
 **/airflow/config/
 !.gitkeep
-**/airflow/dags/
-!.gitkeep
 **/airflow/plugins/
 !.gitkeep
 **/airflow/logs/

src/airflow/dags/dag-inference.py

@@ -0,0 +1,32 @@
+from airflow import DAG
+from airflow.providers.docker.operators.docker import DockerOperator
+from datetime import datetime, timedelta
+
+default_args = {
+    'owner': 'airflow',
+    'retries': 1,
+    'retry_delay': timedelta(minutes=5),
+}
+
+with DAG(
+    dag_id='weekly_inference_dag',
+    default_args=default_args,
+    description='Trigger ML inference every week',
+    schedule_interval='0 9 * * 1',  # Every Monday at 9:00 AM
+    start_date=datetime(2024, 1, 1),
+    catchup=False,
+    tags=['ml', 'neo4j', 'inference'],
+) as dag:
+
+    inference_task = DockerOperator(
+        task_id='inference',
+        image='ghcr.io/sdsc-ordes/open-pulse-airflow:latest',
+        auto_remove=True,
+        environment={
+            'INFERENCE_URL': Variable.get("INFERENCE_URL"),
+            'NEO4J_DATABASE': Variable.get("NEO4J_DATABASE"),
+            'API_TOKEN': Variable.get("API_TOKEN"), 
+        },
+    )
+
+    inference_task

src/airflow/src/env_variables.dist.json

@@ -0,0 +1,5 @@
+{
+    "NEO4J_DATABASE": "",
+    "INFERENCE_URL": "",
+    "API_TOKEN": ""
+}

src/airflow/src/inference.py

@@ -2,7 +2,7 @@
 import logging
 import os
 
-BASE_URL = os.getenv('ML_BASE_URL', 'https://your-api.com')
+BASE_URL = os.getenv('INFERENCE_URL')
 NEO4J_DB = os.getenv('NEO4J_DATABASE')
 API_TOKEN = os.getenv('API_TOKEN')