docker-compose-dev.yml

x-elasticsearch-image: &elasticsearch-image
  docker-registry.wikimedia.org/repos/search-platform/cirrussearch-opensearch-image:v1.3.20-12
x-memcached: &memcached-image
  memcached:1.6.39
x-mysql-image: &mysql-image
  mariadb:11.6
x-redis: &redis-image
  redis:7
x-redis-jobrunner: &redis-jobrunner-image
  ghcr.io/mardi4nfdi/redis-jobrunner-staging:latest
x-wikibase: &wikibase-image
  ghcr.io/mardi4nfdi/wikibase-staging:latest
x-wikibase-apache: &apache-image
  ghcr.io/mardi4nfdi/apache:staging
x-wdqs-image: &wdqs-image
  ghcr.io/wmde/wikibase/wdqs:dev-9186217118
x-wdqs-frontend-image: &wdqs-frontend-image
  ghcr.io/wmde/wikibase/wdqs-frontend:dev-9186217118
x-wdqs-proxy-image: &wdqs-proxy-image
  ghcr.io/wmde/wikibase/wdqs-proxy:dev-9186217118
x-mardi-importer-image: &mardi-importer-image
  ghcr.io/mardi4nfdi/docker-importer:main
x-mardi-importer-api-image: &mardi-importer-api-image
  ghcr.io/mardi4nfdi/importer-api:main

x-common-variables: &wikibase_variables
  WIKIBASE_SCHEME: ${WIKIBASE_SCHEME:-http}
  WIKIBASE_HOST: ${WIKIBASE_HOST:-apache-wikibase}
  WIKIBASE_PORT: ${WIKIBASE_PORT:-8080}
  DB_SERVER: ${DB_SERVER:-mysql.svc:3306}
  DB_NAME: ${DB_NAME:-my_wiki}
  DB_USER: ${DB_USER:-sqluser}
  DB_PASS: ${DB_PASS:-change-this-password}
  INITIALIZE_DB: ${INITIALIZE_DB:-true}
  MW_ELASTIC_HOST: ${MW_ELASTIC_HOST:-elasticsearch.svc}
  MW_ELASTIC_PORT: ${MW_ELASTIC_PORT:-9200}
  MW_REDIS_HOST: ${MW_REDIS_HOST:-redis}
  MW_REDIS_PORT: ${MW_REDIS_PORT:-6379}
  MW_WG_JOB_RUN_RATE: 0
  DEPLOYMENT_ENV: prod
  WIKIBASE_PINGBACK: false
  MW_WG_ENABLE_UPLOADS: false
  MW_ADMIN_NAME: ${MW_ADMIN_NAME:-admin}
  MW_ADMIN_PASS: ${MW_ADMIN_PASS:-change-this-password}
  MW_ADMIN_EMAIL: ${MW_ADMIN_EMAIL:-admin@example.com}
  MW_WG_SECRET_KEY: ${MW_SECRET_KEY:-some-secret-key}

volumes:
  elasticsearch-data:
  mysql-data:
  wdqs-data:

services:
  apache:
    image: *apache-image
    container_name: apache-wikibase
    links:
      - mysql
      - wikibase
    depends_on:
      - mysql
      - wikibase
    ports:
      - "8080:80"
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost/w/api.php"]
      interval: 30s
      timeout: 10s
      retries: 5
    environment:
      <<: *wikibase_variables
      WIKIBASE_HOST: mardi-wikibase

  wikibase:
    image: *wikibase-image
    container_name: mardi-wikibase
    links:
      - mysql
    depends_on:
      - mysql
    restart: unless-stopped
    networks:
      default:
          aliases:
           - mardi-wikibase
    environment:
      <<: *wikibase_variables
      
  mysql:
    image: *mysql-image
    container_name: mardi-mysql
    restart: unless-stopped
    volumes:
      - mysql-data:/var/lib/mysql
      - ./my.cnf:/etc/mysql/conf.d/my.cnf
    environment:
      MYSQL_DATABASE: ${DB_NAME}
      MARIADB_USER: ${DB_USER}
      MARIADB_PASSWORD: ${DB_PASS}
      MARIADB_ROOT_PASSWORD: ${DB_ROOT_PASS:-my-secret-pw}
      
    healthcheck:
      test: ["CMD", "mariadb-admin", "ping", "-h", "localhost", "-u", "${DB_USER}", "-p${DB_PASS}"]
      timeout: 20s
      retries: 10
      interval: 10s
      start_period: 30s
    networks:
      default:
        aliases:
         - mysql.svc

  elasticsearch:
    image: *elasticsearch-image
    container_name: elasticsearch
    restart: unless-stopped
    volumes:
      - elasticsearch-data:/usr/share/elasticsearch/data
    networks:
      default:
        aliases:
         - elasticsearch.svc
    environment:
      discovery.type: single-node
      ES_JAVA_OPTS: "-Xms512m -Xmx512m"

  memcached:
    container_name: memcached
    restart: unless-stopped
    image: *memcached-image

  redis:
    container_name: redis
    restart: unless-stopped
    image: *redis-image

  redis-jobrunner:
    image: *redis-jobrunner-image
    container_name: redis-jobrunner
    depends_on:
    - wikibase
    command: 
    - php
    - /jobrunner/redisJobRunnerService
    - "--config-file=/jobrunner/config.json"
    volumes:
    - ./jobrunner/dev-config.json:/jobrunner/config.json:ro
    environment:
      <<: *wikibase_variables

  redis-rescheduler:
    image: *redis-jobrunner-image
    container_name: redis-rescheduler
    depends_on:
    - wikibase
    command: 
    - php
    - /jobrunner/redisJobChronService
    - "--config-file=/jobrunner/config.json"
    volumes:
    - ./jobrunner/dev-config.json:/jobrunner/config.json:ro
    environment:
      <<: *wikibase_variables

  wdqs-frontend:
    image: *wdqs-frontend-image
    container_name: wdqs-frontend
    restart: unless-stopped
    volumes:
      - ./mediawiki/images/MaRDI_Logo_L_5_rgb_50p.svg:/usr/share/nginx/html/logo.svg
      - ./mediawiki/images/favicon.ico:/usr/share/nginx/html/favicon.ico
    depends_on:
      - wdqs-proxy
      - wikibase
    ports:
      - "8834:80"
    networks:
      default:
        aliases:
         - wdqs-frontend.svc
    environment:
      - WIKIBASE_HOST=mardi-wikibase
      - WDQS_HOST=wdqs-proxy.svc
      - BRAND_TITLE=MaRDIQueryService

  wdqs:
    image: *wdqs-image
    container_name: wdqs
    restart: unless-stopped
    command: /runBlazegraph.sh
    volumes:
      - wdqs-data:/wdqs/data
      - ./wdqs/prefixes.conf:/wdqs/prefixes.conf:ro
      - ./wdqs/allowlist.txt:/wdqs/allowlist.txt:ro
    networks:
      default:
        aliases:
         - wdqs.svc
    environment:
      - WIKIBASE_HOST=${WIKIBASE_HOST:-mardi-wikibase}
      - WIKIBASE_SCHEME=${WIKIBASE_SCHEME:-http}
      - WDQS_HOST=wdqs.svc
      - WDQS_PORT=9999
      - BLAZEGRAPH_OPTS=-Dorg.wikidata.query.rdf.tool.rdf.RdfRepository.timeout=3600
    expose:
      - 9999

  wdqs-proxy:
    image: *wdqs-proxy-image
    container_name: wdqs-proxy
    restart: unless-stopped
    environment:
      - PROXY_PASS_HOST=wdqs.svc:9999
      - PROXY_MAX_QUERY_MILLIS=3600000
    depends_on:
      - wdqs
    networks:
      default:
        aliases:
         - wdqs-proxy.svc

  wdqs-updater:
    image: *wdqs-image
    container_name: wdqs-updater
    restart: unless-stopped
    command: /runUpdate.sh
    depends_on:
    - wdqs
    - wikibase
    networks:
      default:
        aliases:
         - wdqs-updater.svc
    environment:
      - WIKIBASE_HOST=${WIKIBASE_HOST:-mardi-wikibase}
      - WIKIBASE_SCHEME=${WIKIBASE_SCHEME:-http}
      - WDQS_HOST=wdqs.svc
      - WDQS_PORT=9999
      - UPDATER_OPTS=-Dorg.wikidata.query.rdf.tool.wikibase.WikibaseRepository.timeout=30000

  wikibase-setup:
    image: *wikibase-image
    container_name: wikibase-setup
    restart: "no"
    depends_on:
      apache:
        condition: service_healthy
    volumes:
      - ./dev:/setup-scripts
    entrypoint: ["/setup-scripts/setup.sh"]
    environment:
      <<: *wikibase_variables
  
  mysql-setup:
    image: *mysql-image
    container_name: mysql-setup
    restart: "no"
    depends_on:
      mysql:
        condition: service_healthy
      apache:
        condition: service_healthy
    volumes:
      - ./dev:/setup-scripts
    command: >
      bash -c "
        echo 'Running MariaDB importer setup...'
        mariadb -h mysql.svc -u$$DB_USER -p$$DB_PASS < /setup-scripts/setup.sql
        echo 'Finished!'
      "
    environment:
      DB_USER: root
      DB_PASS: ${DB_ROOT_PASS:-my-secret-pw}

  mardi-importer:
    image: *mardi-importer-image
    container_name: mardi-importer
    depends_on:
      - wikibase
      - wikibase-setup
      - mysql-setup
    restart: always
    environment:
      - DB_HOST=${DB_HOST:-mysql.svc}
      - DB_NAME=${DB_IMP_NAME:-wikidata-importer}
      - DB_USER=${DB_USER}
      - DB_PASS=${DB_PASS}
      - IMPORTER_AGENT="mardi-importer"
      - MEDIAWIKI_API_URL=http://apache-wikibase/w/api.php
      - SPARQL_ENDPOINT_URL=http://wdqs-proxy.svc/bigdata/namespace/wdq/sparql
      - WIKIBASE_URL=http://apache-wikibase
      - IMPORTER_API_URL=http://importer-api
      - WIKIBASE_HOST=localhost
      - WIKIBASE_SCHEME=http
    env_file:
    - ./dev/.env

  mardi-importer-api:
    image: *mardi-importer-api-image
    container_name: importer-api
    restart: unless-stopped
    depends_on:
      - wikibase
      - wikibase-setup
      - mysql-setup
    networks:
      default:
        aliases:
         - importer-api.svc
    environment:
      DB_HOST: ${DB_HOST:-mysql.svc}
      MYSQL_USER: ${DB_USER}
      MYSQL_PASSWORD: ${DB_PASS}
      MYSQL_DATABASE: ${DB_IMP_NAME:-wikidata-importer}

  kafka:
    image: apache/kafka:3.7.0
    profiles: ["events"]
    container_name: kafka
    restart: unless-stopped
    environment:
      - KAFKA_NODE_ID=0
      - CLUSTER_ID=MkU3OEVBNTcwNTJENDM2Qk
      - KAFKA_PROCESS_ROLES=controller,broker
      - KAFKA_CONTROLLER_QUORUM_VOTERS=0@kafka:9093
      
      - KAFKA_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka.svc:9092
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      
      - KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT

      - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
    networks:
      default:
        aliases:
          - kafka.svc

  eventstreams:
    image: docker-registry.wikimedia.org/repos/data-engineering/eventstreams:v0.9.0
    profiles: ["events"]
    container_name: eventstreams
    restart: unless-stopped
    depends_on:
      - kafka
    ports:
      - "8092:8092"
    volumes:
      - ./eventstreams/config.yaml:/srv/service/config.yaml
      - ./eventstreams/stream-config.yaml:/srv/service/stream-config.yaml
    networks:
      default:
        aliases:
          - eventstreams.svc

  eventgate:
    image: docker-registry.wikimedia.org/wikimedia/eventgate-wikimedia:ffd68c0de41e3395e2f8ba9422fbe8824c2a49ff
    profiles: ["events"]
    container_name: eventgate
    restart: unless-stopped
    depends_on:
      - kafka
    ports:
      - "8192:8192"
    environment:
      # This points to the config file inside the container
      - CONFIG_FILE=/srv/service/config.yaml
      - NODE_PATH=/srv/service/node_modules/@wikimedia
      - NODE_TLS_REJECT_UNAUTHORIZED=0
    volumes:
      - ./eventgate/config.yaml:/srv/service/config.yaml
      - ./eventgate/schemas:/srv/service/schemas
    networks:
      default:
        aliases:
          - eventgate.svc

networks:                                
  default:                               
    driver: bridge                       
    driver_opts:                         
      com.docker.network.driver.mtu: 1450