feat: airflow 3.0 upgrade

Author: ividito

airflow_services/Dockerfile

@@ -1,6 +1,6 @@
-FROM --platform=linux/amd64 apache/airflow:slim-2.10.5-python3.11
+FROM --platform=linux/amd64 apache/airflow:slim-3.0.2-python3.12
 
-ARG AIRFLOW_VERSION=2.10.5
+ARG AIRFLOW_VERSION=3.0.2
 
 USER root
 # `apt-get autoremove` is used to remove packages that were automatically installed to satisfy
@@ -19,7 +19,7 @@ COPY --chown=airflow:airflow airflow_services/requirements-in.txt "${AIRFLOW_HOM
 USER airflow
 
 RUN uv pip install virtualenv\ 
-  && uv pip install --no-cache-dir -r requirements.txt -c "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.11.txt"\
+  && uv pip install --no-cache-dir -r requirements.txt -c "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.12.txt"\
   && uv pip install --no-cache-dir -r requirements-in.txt apache-airflow==${AIRFLOW_VERSION}
 
 ENV PATH=".venv/bin:$PATH"

airflow_worker/Dockerfile

@@ -1,7 +1,7 @@
 FROM --platform=linux/amd64 ghcr.io/osgeo/gdal:ubuntu-small-3.6.4
 COPY --from=ghcr.io/astral-sh/uv:0.7.3 /uv /uvx /bin/
 
-ARG AIRFLOW_VERSION=2.10.5
+ARG AIRFLOW_VERSION=3.0.2
 ARG UNAME=airflow
 ARG UID=50000
 ARG GID=0
@@ -15,7 +15,7 @@ WORKDIR /opt/airflow
 RUN chown $UNAME:$GID /opt/airflow
 
 RUN apt-get -y update \
-        && apt-get install -y --no-install-recommends gcc libc6-dev libcurl4-openssl-dev libssl-dev python3.11 python3.11-venv python3.11-dev git \
+        && apt-get install -y --no-install-recommends gcc libc6-dev libcurl4-openssl-dev libssl-dev git \
         && apt-get autoremove -yqq --purge \
         && apt-get clean \
         && rm -rf /var/lib/apt/lists/*
@@ -25,10 +25,11 @@ USER airflow
 COPY --chown=airflow:airflow airflow_worker/requirements.txt "${AIRFLOW_HOME}/requirements.txt"
 COPY --chown=airflow:airflow airflow_worker/requirements-in.txt "${AIRFLOW_HOME}/requirements-in.txt"
 
-RUN uv venv --python python3.11
+RUN uv python install 3.12
+RUN uv venv --python 3.12
 RUN uv pip install virtualenv\
-    && uv pip install "apache-airflow[celery,amazon]==${AIRFLOW_VERSION}" -c "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.11.txt"\
-    && uv pip install --no-cache-dir -r requirements.txt -c "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.11.txt"\
+    && uv pip install "apache-airflow[celery,amazon]==${AIRFLOW_VERSION}" -c "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.12.txt"\
+    && uv pip install --no-cache-dir -r requirements.txt -c "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-3.12.txt"\
     && uv pip install --no-cache-dir -r requirements-in.txt apache-airflow==${AIRFLOW_VERSION}
 
 ENV PATH=".venv/bin:$PATH"

infrastructure/configuration/airflow.cfg

@@ -25,9 +25,8 @@ client_secret = 8cbd483d2cb4e73599dffba93dbd0295ef0830c5
 oauth_callback_route = /home
 allowed_teams = VEDA
 
-[webserver]
+[api_server]
 authenticate = True
-auth_backends = airflow.contrib.auth.backends.github_enterprise_auth
 dag_default_view = grid
 expose_config = true
 dag_orientation = TB

infrastructure/main.tf

@@ -26,7 +26,7 @@ module "rds_backups" {
 
 
 module "sma-base" {
-  source                         = "https://github.com/NASA-IMPACT/self-managed-apache-airflow/releases/download/v1.1.15/self-managed-apache-airflow.zip"
+  source                         = "https://github.com/NASA-IMPACT/self-managed-apache-airflow/releases/download/v1.2.0-rc1/self-managed-apache-airflow.zip"
   project                        = var.project_name
   airflow_db                     = var.airflow_db
   fernet_key                     = var.fernet_key