Nebula: Introduce Nebula Transcription Service

.github/workflows/nebula_test.yml

@@ -73,12 +73,22 @@ jobs:
         run: poetry install
         working-directory: nebula
 
+      - name: Use example config as test config
+        run: cp application_local.example.nebula.yml application_local.nebula.yml
+        working-directory: nebula
+
+      - name: Clean old coverage data
+        run: |
+          rm -f .coverage
+          rm -f coverage.xml
+        working-directory: nebula
+
       - name: Run Tests
         run: poetry run coverage run -m pytest
         working-directory: nebula
 
       - name: Generate Coverage Report
-        run: poetry run coverage xml
+        run: poetry run coverage xml --ignore-errors
         working-directory: nebula
 
       #- name: Upload Coverage Report

iris/src/iris/web/routers/pipelines.py

@@ -129,6 +129,7 @@ def run_exercise_chat_pipeline(
         # Additional validation for ChatGPT wrapper variant
         validate_pipeline_variant(dto.settings, ChatGPTWrapperPipeline)
         thread = Thread(target=run_chatgpt_wrapper_pipeline_worker, args=(dto, variant))
+
     else:
         thread = Thread(
             target=run_exercise_chat_pipeline_worker,

nebula/.flake8

@@ -1,5 +1,6 @@
 [flake8]
 max-line-length = 120
+extend-ignore = E203
 exclude =
     .git,
     __pycache__,

nebula/.gitignore

@@ -177,3 +177,10 @@ cython_debug/
 .idea/
 
 .DS_Store
+
+# Ignore transcription temp files
+nebula/src/nebula/transcript/temp/
+
+# Ignore real config files
+nebula/llm_config.nebula.yml
+nebula/application_local.nebula.yml

nebula/README.md

@@ -0,0 +1,5 @@
+# Nebula
+
+This is the central orchestration repository for all Nebula services.
+
+- [Transcript README](./src/nebula/transcript/README.md)

nebula/application_local.example.nebula.yml

@@ -0,0 +1,2 @@
+api_keys:
+  - token: "nebula-secret"

nebula/docker-compose.yml

@@ -0,0 +1,47 @@
+version: "3.9"
+
+services:
+  transcriber:
+    build:
+      context: .
+      dockerfile: docker/transcriber/Dockerfile
+    container_name: nebula-transcriber
+    expose:
+      - "5000" # interne Kommunikation mit Envoy
+    environment:
+      - APPLICATION_YML_PATH=/app/application_local.nebula.yml
+      - LLM_CONFIG_PATH=/app/llm_config.nebula.yml
+    volumes:
+      - ./application_local.nebula.yml:/app/application_local.nebula.yml:ro
+      - ./llm_config.nebula.yml:/app/llm_config.nebula.yml:ro
+      - ./temp:/app/temp
+    restart: unless-stopped
+
+  faq:
+    build:
+      context: .
+      dockerfile: docker/faq/Dockerfile
+    container_name: ${FAQ_SERVICE_NAME}
+    expose:
+      - "${FAQ_SERVICE_PORT}"
+    environment:
+      - APPLICATION_YML_PATH=/app/application_local.nebula.yml
+      - LLM_CONFIG_PATH=/app/llm_config.nebula.yml
+      - FAQ_SERVICE_PORT=${FAQ_SERVICE_PORT}
+    volumes:
+      - ./application_local.nebula.yml:/app/application_local.nebula.yml:ro
+      - ./llm_config.nebula.yml:/app/llm_config.nebula.yml:ro
+    restart: unless-stopped
+
+  envoy:
+    image: envoyproxy/envoy:v1.30-latest
+    container_name: envoy
+    volumes:
+      - ./envoy.yaml:/etc/envoy/envoy.yaml:ro
+    ports:
+      - "8000:8000" # HTTP for FastAPI
+      - "50051:50051" # gRPC
+      - "9901:9901" # Admin interface
+    depends_on:
+      - transcriber
+      - faq

nebula/docker/faq/Dockerfile

@@ -0,0 +1,44 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install Poetry and project dependencies
+COPY pyproject.toml poetry.lock ./
+RUN pip install poetry && poetry install --only main --no-root
+
+# Copy source code and config
+COPY src/nebula ./nebula
+COPY application_local.nebula.yml .
+COPY llm_config.nebula.yml .
+
+# COMPILE PROTO FROM ROOT CONTEXT!
+# This makes relative imports (from . import faq_pb2) work inside grpc_stubs
+WORKDIR /app
+
+# Compile .proto file into grpc_stubs
+# Compile the proto file
+RUN poetry run python -m grpc_tools.protoc \
+    -I=nebula/protos \
+    --python_out=nebula/grpc_stubs \
+    --grpc_python_out=nebula/grpc_stubs \
+    nebula/protos/faq.proto && \
+    echo "✅ Proto compiled!" && \
+    ls -l nebula/grpc_stubs
+
+RUN sed -i 's/^import faq_pb2/import nebula.grpc_stubs.faq_pb2/' nebula/grpc_stubs/faq_pb2_grpc.py
+
+
+# Debug output: list grpc_stubs content
+RUN echo "Contents of grpc_stubs:" && ls -l nebula/grpc_stubs
+
+# Ensure grpc_stubs is a proper Python package
+RUN touch nebula/grpc_stubs/__init__.py
+
+# Set PYTHONPATH so packages can be resolved
+ENV PYTHONPATH=/app
+
+# Expose the gRPC port
+EXPOSE 50052
+
+# Start the FAQ gRPC server
+CMD ["poetry", "run", "python", "nebula/faq/faq_server.py"]

nebula/docker/gateway/Dockerfile

@@ -0,0 +1,34 @@
+# docker/gateway/Dockerfile
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install Poetry + dependencies
+COPY pyproject.toml poetry.lock ./
+RUN pip install poetry && poetry install --only main --no-root
+
+# Copy source code (außer protos und stubs)
+COPY src/nebula ./nebula
+COPY application_local.nebula.yml ./
+
+# Copy proto definitions
+COPY src/nebula/protos ./nebula/protos
+
+# Compile proto to grpc_stubs. This should be done in the compose later on
+RUN poetry run python -m grpc_tools.protoc \
+    -I=nebula/protos \
+    --python_out=nebula/grpc_stubs \
+    --grpc_python_out=nebula/grpc_stubs \
+    nebula/protos/faq.proto && \
+    echo "✅ Proto compiled!" && \
+    ls -l nebula/grpc_stubs
+
+
+RUN sed -i 's/^import faq_pb2/import nebula.grpc_stubs.faq_pb2/' nebula/grpc_stubs/faq_pb2_grpc.py
+
+
+# Set Python path so modules are discoverable
+ENV PYTHONPATH=/app
+
+# Start FastAPI (oder später kombiniert mit gRPC)
+CMD ["poetry", "run", "uvicorn", "nebula.gateway.main:app", "--host", "0.0.0.0", "--port", "8000"]

nebula/docker/transcriber/Dockerfile

@@ -0,0 +1,36 @@
+# Use Python 3.12 slim image
+FROM python:3.12-slim
+
+# System dependencies
+RUN apt-get update && apt-get install -y \
+    ffmpeg \
+    tesseract-ocr \
+    libgl1 \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory inside the container
+WORKDIR /app
+
+# Copy dependency files first
+COPY pyproject.toml poetry.lock ./
+
+# Install poetry
+RUN pip install poetry
+
+# Install only main dependencies
+RUN poetry install --only main --no-root
+
+# Copy source code and config files
+COPY src/nebula ./src/nebula
+COPY application_local.nebula.yml .
+COPY llm_config.nebula.yml .
+
+# Set PYTHONPATH so Python can find src/nebula
+ENV PYTHONPATH=/app/src
+
+# Expose FastAPI port
+EXPOSE 5000
+
+# Run app
+CMD ["poetry", "run", "uvicorn", "nebula.transcript.app:app", "--host", "0.0.0.0", "--port", "5000"]

nebula/envoy.yaml

@@ -0,0 +1,119 @@
+static_resources:
+  listeners:
+    # HTTP listener (for services like FastAPI)
+    - name: listener_http
+      address:
+        socket_address: { address: 0.0.0.0, port_value: 8000 }
+      filter_chains:
+        - filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                stat_prefix: ingress_http
+                route_config:
+                  name: http_route
+                  virtual_hosts:
+                    - name: fastapi
+                      domains: ["*"]
+                      routes:
+                        - match: { prefix: "/transcribe" }
+                          route: { cluster: transcriber }
+
+                        # Add new HTTP service routes here if needed
+                        # - match: { prefix: "/another-api" }
+                        #   route: { cluster: another_http_service }
+
+                http_filters:
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
+    # gRPC listener on port 50051
+    - name: listener_grpc
+      address:
+        socket_address:
+          address: 0.0.0.0
+          port_value: 50051
+      filter_chains:
+        - filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                codec_type: AUTO
+                stat_prefix: ingress_grpc
+                route_config:
+                  name: grpc_route
+                  virtual_hosts:
+                    - name: grpc_services
+                      domains: ["*"]
+                      routes:
+                        - match:
+                            prefix: "/de.tum.cit.aet.artemis.nebula.FAQService/"
+                          route:
+                            cluster: faq
+
+                        # Add a new gRPC service route by its fully-qualified service name
+                        # - match:
+                        #     prefix: "/lectureservice.LectureService/"
+                        #   route:
+                        #     cluster: lecture
+
+                http_filters:
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
+  clusters:
+    # Cluster for the FastAPI-based transcriber service
+    - name: transcriber
+      connect_timeout: 0.5s
+      type: logical_dns
+      lb_policy: round_robin
+      load_assignment:
+        cluster_name: transcriber
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: transcriber
+                      port_value: 5000
+
+    # Cluster for the FAQ gRPC service
+    - name: faq
+      connect_timeout: 0.5s
+      type: logical_dns
+      lb_policy: round_robin
+      http2_protocol_options: {}  # Required for gRPC
+      load_assignment:
+        cluster_name: faq
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: nebula-faq
+                      port_value: 50052
+
+    # Add a new gRPC cluster for another microservice (e.g., lecture)
+    # - name: lecture
+    #   connect_timeout: 0.5s
+    #   type: logical_dns
+    #   lb_policy: round_robin
+    #   http2_protocol_options: {}  # Required for gRPC
+    #   load_assignment:
+    #     cluster_name: lecture
+    #     endpoints:
+    #       - lb_endpoints:
+    #           - endpoint:
+    #               address:
+    #                 socket_address:
+    #                   address: nebula-lecture  # Docker container name or DNS
+    #                   port_value: 50052        # Port used by the lecture gRPC server
+
+admin:
+  access_log_path: /tmp/envoy_admin.log
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 9901