services.dev.yml

#---------------------------------------------------------------------------#
# Common snippets / anchors                                                 #
#---------------------------------------------------------------------------#

x-nifi-logging-common: &nifi-logging-common
  driver: "json-file"
  options:
    max-size: ${NIFI_DOCKER_LOG_SIZE_PER_FILE:-250m}
    max-file: ${NIFI_DOCKER_LOG_NUM_FILES:-10}

x-all-env: &all-env
  - ./project.env
  - ./general.env
  - ./nifi.env
  - ./gitea.env
  - ./nginx.env
  - ./telemetry.env
  - ./database.env
  - ./elasticsearch.env
  - ./network_settings.env
  - ../security/env/users_nifi.env
  - ../security/env/users_database.env
  - ../security/env/users_nginx.env
  - ../security/env/users_elasticsearch.env
  - ../security/env/certificates_general.env
  - ../security/env/certificates_elasticsearch.env
  - ../security/env/certificates_nifi.env

x-es-env: &es-env
  - ./network_settings.env
  - ./elasticsearch.env
  - ../security/env/users_elasticsearch.env
  - ../security/env/certificates_elasticsearch.env

x-db-env: &db-env
  - ./database.env
  - ../security/env/users_database.env

x-common-hosts: &common-hosts
  - ${ELASTICSEARCH_1_HOST_NAME:-test-1:0.0.0.0}
  - ${ELASTICSEARCH_2_HOST_NAME:-test-2:0.0.0.0}
  - ${ELASTICSEARCH_3_HOST_NAME:-test-3:0.0.0.0}
  - ${KIBANA_HOST_NAME:-test-4:0.0.0.0}
  - ${NIFI_HOST_NAME:-test-5:0.0.0.0}

x-common-ulimits: &common-ulimits
  ulimits:
    nofile:
      soft: 65535
      hard: 65535
    nproc: 65535
    memlock:
      soft: -1
      hard: -1

x-nifi-common: &nifi-common
  <<: *common-ulimits
  restart: unless-stopped
  env_file: *all-env
  extra_hosts: *common-hosts
  networks:
    - cognet

x-nifi-volumes: &nifi-volumes
  # Drivers
  - ../nifi/drivers:/opt/nifi/drivers:ro

  # User overrides bundled in the image
  - ../nifi/user_scripts:/opt/nifi/user_scripts:rw
  - ../nifi/user_schemas:/opt/nifi/user_schemas:ro

  # Python processors (NiFi 2.x)
  - ../nifi/user_python_extensions:/opt/nifi/nifi-current/python_extensions:rw

  # Security certificates
  - ../security:/security:ro

  # Security helper scripts
  - ../security/scripts/nifi_create_single_user_auth.sh:/opt/nifi/nifi-current/security_scripts/nifi_create_single_user_auth.sh:ro

  # NiFi configuration
  - ../nifi/conf/:/opt/nifi/nifi-current/conf/:rw

  # Ingest data directory
  - ./${NIFI_DATA_PATH:-../data/}:/data/:rw

  # MedCAT models
  - ./${NIFI_MEDCAT_SERVICE_MODEL_PRODUCTION_PATH:-../services/cogstack-nlp/medcat-service/models/}:/opt/models:rw

  # NiFi repositories/state
  - nifi-vol-logs:/opt/nifi/nifi-current/logs
  - nifi-vol-provenance:/opt/nifi/nifi-current/provenance_repository
  - nifi-vol-database:/opt/nifi/nifi-current/database_repository
  - nifi-vol-flowfiles:/opt/nifi/nifi-current/flowfile_repository
  - nifi-vol-content:/opt/nifi/nifi-current/content_repository
  - nifi-vol-state:/opt/nifi/nifi-current/state

  # Flowfile error output
  - nifi-vol-errors:/opt/nifi/pipeline/flowfile-errors

#---------------------------------------------------------------------------#
# Used services                                                             #
#---------------------------------------------------------------------------#
services:

#---------------------------------------------------------------------------#
# NiFi webapp                                                               #
#---------------------------------------------------------------------------#
  nifi:
    <<: *nifi-common
    build:
       context: ..
       dockerfile: nifi/Dockerfile
    container_name: cogstack-nifi
    hostname: nifi
    shm_size: ${NIFI_DOCKER_SHM_SIZE:-"1g"}
    environment:
      - PYTHONPATH=${NIFI_PYTHONPATH:-/opt/nifi/nifi-current/python/framework}
      - JVM_OPTS="${NIFI_JVM_OPTS:--XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+ParallelRefProcEnabled -Djava.security.egd=file:/dev/./urandom}"
    deploy:
      resources:
        limits:
          cpus: "${NIFI_DOCKER_CPU_MAX}"
          memory: "${NIFI_DOCKER_RAM}"
        reservations:
          cpus: "${NIFI_DOCKER_CPU_MIN}"
          memory: "${NIFI_DOCKER_RAM}"
    volumes: *nifi-volumes

    # INFO : Uncomment the below line to generate your own USERNAME and PASSWORD,
    #        a bit messy this way as you will need to copy the credentials back
    #        to the "login-identity-providers.xml" section.
    # entrypoint: bash -c "/opt/nifi/nifi-current/bin/nifi.sh set-single-user-credentials admin admincogstacknifi"
    tty: true
    ports:
      - "${NIFI_OUTPUT_PORT:-8082}:${NIFI_INTERNAL_PORT:-8443}"
      - "${NIFI_INPUT_SOCKET_PORT:-10000}"
    logging: *nifi-logging-common
  
  nifi-nginx:
    build:
      context: ../services/nginx
      dockerfile: Dockerfile
    container_name: cogstack-nifi-nginx
    restart: always
    shm_size: ${NGINX_SHM_SIZE:-512mb}
    env_file: *all-env
    deploy:
      resources:
        limits:
          cpus: "${NGINX_DOCKER_CPU_MAX}"
          memory: "${NGINX_DOCKER_RAM}"
        reservations:
          cpus: "${NGINX_DOCKER_CPU_MIN}"
          memory: "${NGINX_DOCKER_RAM}"
    volumes:
      - ../services/nginx/sites-enabled:/etc/nginx/sites-enabled:ro
      - ../services/nginx/config/nginx.conf.template:/etc/nginx/templates/nginx.conf.template:ro
      - ../security/certificates:/certificates:ro
    ports:
      - "${NIFI_EXTERNAL_PORT_NGINX:-8443}:${NIFI_INTERNAL_PORT_NGINX:-8443}"
    networks:
      - cognet
    extra_hosts: *common-hosts
    logging: *nifi-logging-common

#---------------------------------------------------------------------------#
# Observability stack                                                       #
#---------------------------------------------------------------------------#
  otel-collector:
    image: ${OTEL_COLLECTOR_DOCKER_IMAGE:-otel/opentelemetry-collector-contrib:0.132.0}
    container_name: cogstack-otel-collector
    restart: unless-stopped
    command: ["--config=/etc/otelcol-contrib/config.yaml"]
    env_file: *all-env
    volumes:
      - ./observability/otel-collector.yaml:/etc/otelcol-contrib/config.yaml:ro
    ports:
      - "${OTEL_COLLECTOR_OTLP_GRPC_PORT:-4317}:4317"
      - "${OTEL_COLLECTOR_OTLP_HTTP_PORT:-4318}:4318"
      - "${OTEL_COLLECTOR_HEALTH_PORT:-13133}:13133"
    networks:
      - cognet
    extra_hosts: *common-hosts
    logging: *nifi-logging-common

  data-prepper:
    image: ${DATA_PREPPER_DOCKER_IMAGE:-opensearchproject/data-prepper:2.11.0}
    container_name: cogstack-data-prepper
    restart: unless-stopped
    env_file: *all-env
    volumes:
      - ./observability/data-prepper/config/data-prepper-config.yaml:/usr/share/data-prepper/config/data-prepper-config.yaml:ro
      - ./observability/data-prepper/pipelines:/usr/share/data-prepper/pipelines:ro
    ports:
      - "${DATA_PREPPER_API_PORT:-4900}:4900"
      - "${DATA_PREPPER_OTEL_TRACE_PORT:-21890}:21890"
    networks:
      - cognet
    extra_hosts: *common-hosts
    logging: *nifi-logging-common

#---------------------------------------------------------------------------#
# Docker named volumes                                                      #
#---------------------------------------------------------------------------#
volumes:
  # NiFi related
  nifi-vol-logs:
    driver: local

  nifi-vol-provenance:
    driver: local

  nifi-vol-database:
    driver: local

  nifi-vol-flowfiles:
    driver: local

  nifi-vol-content:
    driver: local
  
  nifi-vol-state:
    driver: local

  nifi-vol-errors:
    driver: local

#---------------------------------------------------------------------------#
# Docker networks.                                                          #
#---------------------------------------------------------------------------#
networks:
  cognet:
    driver: bridge
    name: cogstack-net