hokusai-data-pipeline/docker-compose.minimal.yml at main · Hokusai-protocol/hokusai-data-pipeline · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
version: '3.8'

services:
  # PostgreSQL database for MLFlow backend
  postgres:
    image: postgres:14-alpine
    container_name: hokusai_postgres
    environment:
      POSTGRES_USER: mlflow
      POSTGRES_PASSWORD: mlflow_password
      POSTGRES_DB: mlflow_db
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U mlflow"]
      interval: 10s
      timeout: 5s
      retries: 5

  # MinIO for S3-compatible artifact storage
  minio:
    image: minio/minio:latest
    container_name: hokusai_minio
    environment:
      MINIO_ROOT_USER: minioadmin
      MINIO_ROOT_PASSWORD: minioadmin123
    command: server /data --console-address ":9001"
    volumes:
      - minio_data:/data
    ports:
      - "9000:9000"
      - "9001:9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  # Create MinIO bucket for MLFlow
  minio-init:
    image: minio/mc:latest
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: >
      /bin/sh -c "
      mc alias set minio http://minio:9000 minioadmin minioadmin123;
      mc mb minio/mlflow-artifacts || true;
      mc policy set public minio/mlflow-artifacts;
      exit 0;
      "

  # MLFlow tracking server
  mlflow-server:
    build:
      context: .
      dockerfile: Dockerfile.mlflow
    container_name: hokusai_mlflow
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
    environment:
      MLFLOW_S3_ENDPOINT_URL: http://minio:9000
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin123
    command: >
      mlflow server
      --backend-store-uri postgresql://mlflow:mlflow_password@postgres/mlflow_db
      --default-artifact-root s3://mlflow-artifacts/
      --host 0.0.0.0
      --port 5000
    ports:
      - "5001:5000"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:5000/"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Model Registry API service
  model-registry-api:
    build:
      context: .
      dockerfile: Dockerfile.api
    container_name: hokusai_api
    depends_on:
      mlflow-server:
        condition: service_healthy
    environment:
      MLFLOW_TRACKING_URI: http://hokusai_mlflow:5000
      API_HOST: 0.0.0.0
      API_PORT: 8001
      POSTGRES_URI: postgresql://mlflow:mlflow_password@postgres/mlflow_db
    ports:
      - "8001:8001"
    volumes:
      - ./src:/app/src
      - ./tests:/app/tests
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8001/health"]
      interval: 30s
      timeout: 10s
      retries: 3

  # Redis for caching and rate limiting
  redis:
    image: redis:7-alpine
    container_name: hokusai_redis
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 5s
      retries: 5

volumes:
  postgres_data:
  minio_data:
  redis_data:

networks:
  default:
    name: hokusai_network