feat: масштабный рефакторинг и расширение функциональности (#1)

Author: Powermacintosh

.env.test

@@ -12,6 +12,11 @@ TASKS_DB_HOST = db_tasks_app_test
 TASKS_DB_PORT = 5432
 TASKS_DB_NAME = tasks_app_test
 
+# Kafka
+KAFKA_BOOTSTRAP = kafka_test:9092
+KAFKA_TOPIC = task_events_test
+KAFKA_CLIENT_ID = tasks_app_test
+
 # Loki
 LOKI_PORT=3100
 

.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: ["main"]
+    branches: [main, experiment/workflows]
   pull_request:
-    branches: ["main"]
+    branches: [main]
 
 jobs:
   tests:
@@ -17,8 +17,8 @@ jobs:
       - name: Build and start test services
         run: docker compose -f docker-compose.test.yml --env-file ./tasks/.env.test up -d --build
 
-      - name: Run pytest inside tasks_app_test
-        run: docker compose -f docker-compose.test.yml --env-file ./tasks/.env.test exec -T tasks_app_test pytest -v
+      - name: Run tests
+        run: docker compose -f docker-compose.test.yml --env-file ./tasks/.env.test exec -T tasks_app_test pytest
 
       - name: Stop and remove test services
         if: always()

Makefile

@@ -8,6 +8,6 @@ build:
 	docker compose up --build -d
 
 test:
-	docker compose -f docker-compose.test.yml up --build -d --remove-orphans
+	docker compose -f docker-compose.test.yml up --build -d
 	docker compose -f docker-compose.test.yml exec -it tasks_app_test bash -c "pytest -v"
-	docker compose -f docker-compose.test.yml down
+	docker compose -f docker-compose.test.yml down -v --remove-orphans

README.md

@@ -16,6 +16,9 @@
 - **Loki** - система логирования
 - **Grafana** - система мониторинга
 - **Promtail** - инструмент для сбора логов
+- **Kafka** - система обмена сообщениями
+- **Kafka-UI** - визуальный интерфейс для работы с Kafka
+- **Locust** - инструмент для нагрузочного тестирования
 
 ## 🚀 Запуск проекта
 
@@ -35,9 +38,7 @@
    cd microservices-example
    ```
 
-2. Создайте файл `.env` на основе `.env.test`:
-
-3. Запустите приложение с помощью Makefile:
+2. Запустите приложение с помощью Makefile:
 
 ```bash
 make build
@@ -51,29 +52,54 @@ make test
 
 ## 📚 Документация API
 
-После запуска документация API будет доступна по адресу:
+После запуска документация API Gateway будет доступна по адресу:
 
 - Swagger UI: http://localhost:5000/docs/
   ![gateway/static/swagger-custom.png](gateway/static/swagger-custom.png)
 
-## 🔧 Настройка окружения
+Также API Gateway предоставляет возможность работы с API через GraphQL:
+
+- GraphQL UI: http://localhost:5000/api/v1/graphql
+  ![gateway/static/graphql-interface.png](gateway/static/graphql-interface.png)
+
+## 🔧 Настройка окружения для разработки
 
-Создайте файл `.env` в корне проекта и в каждом сервисе
+Создайте файл `.env` в корне сервиса
 `gateway/`, `tasks/`.
 Аналогично файлу `.env.test`.
 
 ```env
 MODE = DEVELOPMENT
 ```
 
-## 📊 Логирование
+## 📊 Логирование в Grafana
 
 Логи приложения отправляются в Loki и доступны через Grafana.
 **Grafana**: `http://localhost:3010`
 
+`Логин` - admin
+`Пароль` - admin (при первом входе)
+
 ### Подключение Loki к Grafana
 
-**Loki-connection-url**: `http://loki:3100`
+#### Путь для подключения Loki к Grafana
+
+`Connections` => `Loki` => `Add new data source`
+
+- **Connection-url**: `http://loki:3100`
+- ⚙️ **Save & Test**
+
+#### Путь для просмотра логов в Grafana
+
+`Drilldown` => `Logs`
+
+## 🎯 Подключение к Kafka-UI
+
+**Kafka-UI**: `http://localhost:8080`
+
+## 🧪 Запуск нагрузочного тестирования Locust
+
+**Locust**: `http://localhost:8089`
 
 ## 📄 Лицензия
 

docker-compose.test.yml

@@ -18,6 +18,27 @@ services:
     depends_on:
       db_tasks_app_test:
         condition: service_healthy
+      kafka_test:
+        condition: service_healthy
+
+  tasks_app_worker_test:
+    build:
+      context: tasks/
+    container_name: tasks_app_worker_test
+    env_file:
+      - ./tasks/.env.test
+    environment:
+      SERVICE_NAME: tasks_app_test
+      KAFKA_BOOTSTRAP: kafka_test:9092
+    entrypoint: ["sh", "-c"]
+    command:
+      - |
+        python worker.py
+    depends_on:
+      db_tasks_app_test:
+        condition: service_healthy
+      kafka_test:
+        condition: service_healthy
 
   db_tasks_app_test:
     image: postgres:17
@@ -38,5 +59,30 @@ services:
       timeout: 5s
       retries: 5
 
+  kafka_test:
+    image: bitnami/kafka:latest
+    container_name: kafka_test
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_CFG_NODE_ID: 1
+      KAFKA_CFG_PROCESS_ROLES: "controller,broker"
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka_test:9093"
+      KAFKA_CFG_LISTENERS: "CONTROLLER://:9093,PLAINTEXT://:9092"
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+      KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka_test:9092"
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
+      KAFKA_CFG_LOG_RETENTION_HOURS: "72"  # 3 дня
+    ports:
+      - "9092:9092"
+      - "9093:9093"
+    healthcheck:
+      test: ["CMD", "bash", "-c", "kafka-topics.sh --list --bootstrap-server localhost:9092"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 20s
+
 volumes:
   db_tasks_app_test:

docker-compose.yml

@@ -4,9 +4,11 @@ services:
       context: gateway/
     container_name: gateway_app
     env_file:
-      - ./gateway/.env
+      - ./gateway/.env.docker
     ports:
       - "${GATEWAY_PORT}:${GATEWAY_PORT}"
+    networks:
+      - appnet
     entrypoint: ["sh", "-c"]
     command:
       - |
@@ -16,13 +18,34 @@ services:
     depends_on:
       loki:
         condition: service_started
+      kafka:
+        condition: service_healthy
+  
+  locust_gateway:
+    build:
+      context: gateway/
+    container_name: locust_gateway
+    env_file:
+      - ./gateway/.env.docker
+    ports:
+      - "8089:8089"
+    networks:
+      - appnet
+    entrypoint: ["sh", "-c"]
+    command:
+      - |
+        poetry run locust -f locustfile.py \
+          --host http://gateway_app:${GATEWAY_PORT}
+    depends_on:
+      gateway_app:
+        condition: service_started
 
   tasks_app:
     build:
       context: tasks/
     container_name: tasks_app
     env_file:
-      - ./tasks/.env
+      - ./tasks/.env.docker
     ports:
       - "${TASKS_APP_PORT}"
     entrypoint: ["sh", "-c"]
@@ -38,12 +61,41 @@ services:
         condition: service_healthy
       loki:
         condition: service_started
+      kafka:
+        condition: service_healthy
+    networks:
+      - appnet
+  
+  tasks_app_worker:
+    build:
+      context: tasks/
+    container_name: tasks_app_worker
+    env_file:
+      - ./tasks/.env.docker
+    environment:
+      SERVICE_NAME: tasks_app
+      KAFKA_BOOTSTRAP: kafka:9092
+      KAFKA_GROUP_ID: tasks_app_group
+    entrypoint: ["sh", "-c"]
+    command:
+      - |
+        python worker.py
+    depends_on:
+      db_tasks_app:
+        condition: service_healthy
+      kafka:
+        condition: service_healthy
+    networks:
+      - appnet
+    deploy:
+      restart_policy:
+        condition: on-failure
 
   db_tasks_app:
     image: postgres:17
     container_name: db_tasks_app
     env_file:
-      - ./tasks/.env
+      - ./tasks/.env.docker
     environment:
       POSTGRES_USER: ${TASKS_DB_USER} 
       POSTGRES_PASSWORD: ${TASKS_DB_PASS}
@@ -52,16 +104,64 @@ services:
       - "${TASKS_DB_PORT}"
     volumes:
       - db_tasks_app:/var/lib/postgresql/data
+    networks:
+      - appnet
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${TASKS_DB_USER}"]
       interval: 5s
       timeout: 5s
       retries: 5
-
+  
+  kafka:
+    image: bitnami/kafka:latest
+    container_name: kafka
+    # volumes:
+    #   - kafka_data:/bitnami/kafka
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_CFG_NODE_ID: 1
+      KAFKA_CFG_PROCESS_ROLES: "controller,broker"
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093"
+      KAFKA_CFG_LISTENERS: "CONTROLLER://:9093,PLAINTEXT://:9092"
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+      KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
+      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
+      KAFKA_CFG_LOG_RETENTION_HOURS: "72"  # 3 дня
+    ports:
+      - "9092:9092"
+      - "9093:9093"
+    healthcheck:
+      test: ["CMD", "bash", "-c", "kafka-topics.sh --list --bootstrap-server localhost:9092"]
+      interval: 10s
+      timeout: 5s
+      retries: 6
+      start_period: 20s
+    networks:
+      - appnet
+  
+  kafka-ui:
+    image: provectuslabs/kafka-ui:latest
+    container_name: kafka-ui
+    ports:
+      - "8088:8080"
+    environment:
+      KAFKA_CLUSTERS_0_NAME: local
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
+      KAFKA_CLUSTERS_0_READONLY: "false"
+    depends_on:
+      kafka:
+        condition: service_healthy
+    networks:
+      - appnet
+  
   loki:
     image: grafana/loki:latest
     ports:
       - "${LOKI_PORT}"
+    networks:
+      - appnet
     volumes:
       - ./loki-config.yaml:/etc/loki/local-config.yaml
       - loki_data:/loki
@@ -73,6 +173,8 @@ services:
 
   promtail:
     image: grafana/promtail:latest
+    networks:
+      - appnet
     volumes:
       - ./promtail-config.yaml:/etc/promtail/config.yaml
       - /var/run/docker.sock:/var/run/docker.sock
@@ -87,6 +189,8 @@ services:
 
   grafana:
     image: grafana/grafana:latest
+    networks:
+      - appnet
     ports:
       - "${GRAFANA_PORT}:3000"
     volumes:
@@ -96,7 +200,12 @@ services:
       - GF_SECURITY_ADMIN_PASSWORD=admin
       - GF_USERS_ALLOW_SIGN_UP=false
 
+networks:
+  appnet:
+    driver: bridge
+
 volumes:
   db_tasks_app:
   loki_data:
   grafana_data:
+  # kafka_data: