news_crawler/docker-compose.yml at master · Yariki/news_crawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
services:
  app:
    build:
      context: ./backend
      dockerfile: Dockerfile
    container_name: news_crawler_app
    env_file:
      - ./backend/.env
    environment:
      DATABASE_URL: ${DATABASE_URL}
      ELASTICSEARCH_URL: ${ELASTICSEARCH_URL}
      CORS_ORIGINS: ${CORS_ORIGINS}
      DEFAULT_KEYWORDS: ${DEFAULT_KEYWORDS}
      APP_MODE: ${APP_MODE}
      CRAWL_DELAY: ${CRAWL_DELAY}
      REQUEST_RATE: ${REQUEST_RATE:-10}
      TELEGRAM_API_ID: ${TELEGRAM_API_ID:-}
      TELEGRAM_API_HASH: ${TELEGRAM_API_HASH:-}
      RABBITMQ_URL: ${RABBITMQ_URL:-amqp://guest:guest@rabbitmq:5672/}
      CELERY_TASK_QUEUE: ${CELERY_TASK_QUEUE:-scheduler.checks}
      CHECKER_TIMEOUT_SECONDS: ${CHECKER_TIMEOUT_SECONDS:-30}
      BEAT_TICK_SECONDS: ${BEAT_TICK_SECONDS:-10}
      BEAT_BATCH_SIZE: ${BEAT_BATCH_SIZE:-50}
    command: uvicorn app.main:app --host 0.0.0.0 --port 8000
    ports:
      - "8000:8000"
    depends_on:
      postgres:
        condition: service_healthy
      elasticsearch:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy

  postgres:
    image: postgres:16
    container_name: news_crawler_postgres
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-news_user}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-news_password}
      POSTGRES_DB: ${POSTGRES_DB:-news_monitor}
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "pg_isready -U ${POSTGRES_USER:-news_user} -d ${POSTGRES_DB:-news_monitor}",
        ]
      interval: 10s
      timeout: 5s
      retries: 5

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4
    container_name: news_crawler_elasticsearch
    environment:
      discovery.type: single-node
      xpack.security.enabled: "false"
      ES_JAVA_OPTS: ${ES_JAVA_OPTS:--Xms512m -Xmx512m}
    ports:
      - "9200:9200"
    volumes:
      - elasticsearch_data:/usr/share/elasticsearch/data
    healthcheck:
      test:
        ["CMD-SHELL", "curl -fsS http://localhost:9200 >/dev/null || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 10

  kibana:
    image: docker.elastic.co/kibana/kibana:8.13.4
    container_name: news_crawler_kibana
    environment:
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    ports:
      - "5601:5601"
    depends_on:
      elasticsearch:
        condition: service_healthy

  rabbitmq:
    image: rabbitmq:3.13-management-alpine
    container_name: news_crawler_rabbitmq
    ports:
      - "5672:5672"
      - "15672:15672"
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
      interval: 10s
      timeout: 5s
      retries: 10

  celery_worker:
    build:
      context: ./backend
      dockerfile: Dockerfile
    container_name: news_crawler_celery_worker
    env_file:
      - ./backend/.env
    environment:
      DATABASE_URL: ${DATABASE_URL}
      ELASTICSEARCH_URL: ${ELASTICSEARCH_URL}
      RABBITMQ_URL: ${CELERY_BROKER_URL:-amqp://guest:guest@rabbitmq:5672/}
      CELERY_TASK_QUEUE: ${CELERY_TASK_QUEUE:-scheduler.checks}
      CHECKER_TIMEOUT_SECONDS: ${CHECKER_TIMEOUT_SECONDS:-30}
      TELEGRAM_API_ID: ${TELEGRAM_API_ID:-}
      TELEGRAM_API_HASH: ${TELEGRAM_API_HASH:-}
    command: celery -A app.schedule.celery_app:celery_app worker --loglevel=info -Q ${CELERY_TASK_QUEUE:-scheduler.checks}
    depends_on:
      postgres:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy

  celery_beat:
    build:
      context: ./backend
      dockerfile: Dockerfile
    container_name: news_crawler_celery_beat
    env_file:
      - ./backend/.env
    environment:
      DATABASE_URL: ${DATABASE_URL}
      ELASTICSEARCH_URL: ${ELASTICSEARCH_URL}
      RABBITMQ_URL: ${CELERY_BROKER_URL:-amqp://guest:guest@rabbitmq:5672/}
      CELERY_TASK_QUEUE: ${CELERY_TASK_QUEUE:-scheduler.checks}
      BEAT_TICK_SECONDS: ${BEAT_TICK_SECONDS:-10}
      BEAT_BATCH_SIZE: ${BEAT_BATCH_SIZE:-50}
    command: celery -A app.schedule.celery_beat beat --loglevel=info
    depends_on:
      postgres:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy

volumes:
  postgres_data:
  elasticsearch_data: