CH and friends

Author: kiturutin

lib/config/env.properties

@@ -7,11 +7,6 @@ BROWSERUP_LOCKBOX_MASTER_KEY=439e901f30a3ce31836dbc81e67411e1f84f43491740ca0f91d
 BROWSERUP_GRID_JAVA_API_HOST=grid-java-api
 BROWSERUP_GRID_JAVA_API_PORT=23001
 
-BROWSERUP_MYSQL_HOST=mysql
-BROWSERUP_MYSQL_PORT=23002
-BROWSERUP_MYSQL_PASSWORD=just_testing
-BROWSERUP_MYSQL_USERNAME=browserup
-
 BROWSERUP_S3_MINIO_HOST=minio
 BROWSERUP_S3_MINIO_PORT_1=23003
 BROWSERUP_S3_MINIO_PORT_2=23004
@@ -28,20 +23,11 @@ BROWSERUP_ZOOKEEPER_HOST=zookeeper
 BROWSERUP_ZOOKEEPER_CLIENT_PORT=23008
 BROWSERUP_ZOO_MAX_CLIENT_CNXNS=500
 
-BROWSERUP_REDIS_HOST=redis
-BROWSERUP_REDIS_PORT=23009
+BROWSERUP_CLICKHOUSE_HOST=clickhouse
 
 BROWSERUP_CHRONOGRAF_HOST=chronograf
 BROWSERUP_CHRONOGRAF_PORT=23010
 
-BROWSERUP_INFLUX_DB_HOST=influxdb
-BROWSERUP_INFLUX_DB_PORT=23011
-BROWSERUP_INFLUX_DB_NAME=browserup
-BROWSERUP_INFLUX_DB_USER=browserup
-BROWSERUP_INFLUX_TEST_DB_HOST=influx-test-db
-BROWSERUP_INFLUX_TEST_DB_PORT=23011
-BROWSERUP_INFLUX_DB_PASSWORD=just_testing
-
 BROWSERUP_LOKI_HOST=loki
 
 BROWSERUP_S3_MINIO_HOST_PORT_1=23012
@@ -52,3 +38,6 @@ BROWSERUP_GRAFANA_PASSWORD=just_testing
 
 BROWSERUP_PRIVATE_DNS_NAMESPACE_NAME=browserup-internal.com
 BROWSERUP_CLUSTER_PUBLIC_IP=localhost
+
+BROWSERUP_NATS_URLS=nats://nats:4222,nats://nats-1:4222
+BROWSERUP_NATS_SERVER_LIST=nats:4222,nats-1:4222

lib/docker_compose/docker-compose.yml

@@ -6,20 +6,14 @@ services:
   backend-services:
     image: busybox
     depends_on:
-      - chronograf
-      - influxdb
-      - mysql
       - rabbitmq
-      - redis
       - zookeeper
       - minio
       - grafana
 
   backend-services-nodb:
     image: busybox
     depends_on:
-      - chronograf
-      - influxdb
       - rabbitmq
       - zookeeper
 
@@ -61,12 +55,7 @@ services:
       - BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
       - BROWSERUP_RABBITMQ_QUEUE_PORT=5672
       - BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
-      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=2181
-      - BROWSERUP_INFLUX_DB_HOST=${BROWSERUP_INFLUX_DB_HOST}
-      - BROWSERUP_INFLUX_DB_PORT=8086
-      - BROWSERUP_INFLUX_DB_NAME=browserup
-      - BROWSERUP_INFLUX_DB_USER=browserup
-      - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
+      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=9181
       - BROWSERUP_MINION_VUS=1
       - BROWSERUP_MAX_MINIONS_PER_EC2_INSTANCE=${BROWSERUP_MAX_MINIONS_PER_EC2_INSTANCE}
       - BROWSERUP_IS_CLOUD=false
@@ -88,8 +77,8 @@ services:
       - BROWSERUP_S3_MINIO_HOST_PORT_1=${BROWSERUP_S3_MINIO_PORT_1}
       - BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET=${BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET}
       - BROWSERUP_CLUSTER_NAME=${BROWSERUP_CLUSTER_NAME}
+      - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
     depends_on:
-      - influxdb
       - zookeeper
       - rabbitmq
       - fluentbit
@@ -119,12 +108,7 @@ services:
       - BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
       - BROWSERUP_RABBITMQ_QUEUE_PORT=5672
       - BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
-      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=2181
-      - BROWSERUP_INFLUX_DB_HOST=${BROWSERUP_INFLUX_DB_HOST}
-      - BROWSERUP_INFLUX_DB_PORT=8086
-      - BROWSERUP_INFLUX_DB_NAME=browserup
-      - BROWSERUP_INFLUX_DB_USER=browserup
-      - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
+      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=9181
       - BROWSERUP_MINION_VUS=1
       - BROWSERUP_MINION_IMAGE_VERSION_TAG=${MINION_IMAGE_VERSION_TAG}
       - BROWSERUP_S3_MINIO_ACCESS_KEY_ID=${BROWSERUP_S3_MINIO_ACCESS_KEY_ID}
@@ -138,11 +122,13 @@ services:
       - BROWSERUP_CLUSTER_NAME=${BROWSERUP_CLUSTER_NAME}
       - BROWSERUP_USER_ARTIFACT_CACHE_PATH=/bu/artifactcache
       - BROWSERUP_USE_LOCAL_DOCKER_CACHED_IMAGE=${BROWSERUP_USE_LOCAL_DOCKER_CACHED_IMAGE}
+      - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
     depends_on:
       - zookeeper
       - rabbitmq
       - grid-java-api
       - fluentbit
+      - clickhouse
     logging:
       driver: "fluentd"
       options:
@@ -160,8 +146,9 @@ services:
       - BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
       - BROWSERUP_RABBITMQ_QUEUE_PORT=5672
       - BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
-      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=2181
+      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=9181
       - BROWSERUP_IS_CLOUD=false
+      - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
     depends_on:
       - zookeeper
       - rabbitmq
@@ -174,30 +161,21 @@ services:
 
   webconsole:
     container_name: browserup_webconsole
-    image: docker.io/browserup/webconsole:latest
+    image: docker.io/browserup/webconsole:snapshot
     privileged: true
     security_opt:
       - "seccomp=unconfined"
       - "apparmor=unconfined"
     devices:
       - "/dev/fuse"
     ports:
-      - "${BROWSERUP_WEBCONSOLE_PORT}:3000"
+      - "${BROWSERUP_WEBCONSOLE_PORT}:3100"
     environment:
       - DOCKER_LOGS=true
-      - BROWSERUP_MYSQL_HOST=${BROWSERUP_MYSQL_HOST}
-      - BROWSERUP_MYSQL_USERNAME=browserup
-      - BROWSERUP_MYSQL_PASSWORD=${BROWSERUP_MYSQL_PASSWORD}
-      - BROWSERUP_MYSQL_PORT=3306
-      - BROWSERUP_INFLUX_DB_HOST=${BROWSERUP_INFLUX_DB_HOST}
-      - BROWSERUP_INFLUX_DB_PORT=8086
-      - BROWSERUP_INFLUX_DB_NAME=browserup
-      - BROWSERUP_INFLUX_DB_USER=browserup
-      - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
+      - BROWSERUP_CLICKHOUSE_HOST=${BROWSERUP_CLICKHOUSE_HOST}
       - BROWSERUP_GRID_SERVICE_URL=http://${BROWSERUP_GRID_JAVA_API_HOST}:8080/grid
       - BROWSERUP_LOKI_SERVICE_URL=http://${BROWSERUP_LOKI_HOST}:3100
       - BROWSERUP_IS_CLOUD=false
-      - BROWSERUP_REDIS_URL=redis://${BROWSERUP_REDIS_HOST}:6379/1
       - BROWSERUP_S3_MINIO_ACCESS_KEY_ID=${BROWSERUP_S3_MINIO_ACCESS_KEY_ID}
       - BROWSERUP_S3_MINIO_SECRET_ACCESS_KEY=${BROWSERUP_S3_MINIO_SECRET_ACCESS_KEY}
       - BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET=${BROWSERUP_S3_MINIO_ARTIFACTS_BUCKET}
@@ -211,84 +189,79 @@ services:
       - RAILS_LOG_TO_STDOUT=true
       - RAILS_SERVE_STATIC_FILES=true
       - BROWSERUP_S3_MINIO_HOST_PORT_1=${BROWSERUP_S3_MINIO_PORT_1}
+      - RAILS_MASTER_KEY=6f893b421a180634146cd1eb5f58cc80
+      - PORT=3000
+      - BROWSERUP_WC_RUNNING_AS_SERVER=true
+      - BROWSERUP_NATS_URLS=${BROWSERUP_NATS_URLS}
+      - BROWSERUP_NATS_SERVER_LIST=${BROWSERUP_NATS_SERVER_LIST}
     depends_on:
-      - mysql
-      - influxdb
       - grid-java-api
-      - redis
       - minio
       - fluentbit
     logging:
       driver: "fluentd"
       options:
         fluentd-address: "localhost:24224"
 
-  chronograf:
-    image: chronograf:1.7.16
-    ports:
-      - "${BROWSERUP_CHRONOGRAF_PORT}:8888"
-    depends_on:
-      - influxdb
-      - fluentbit
-    logging:
-      driver: "fluentd"
-      options:
-        fluentd-address: "localhost:24224"
-
-  influxdb:
-    container_name: browserup_influxdb
-    image: influxdb:1.8.10
+  clickhouse:
+    image: 'clickhouse/clickhouse-server:25.1'
+    container_name: clickhouse
+    hostname: clickhouse
     volumes:
-      - influxdb-volume:/var/lib/influxdb
+      - ./resources/click-house/config.xml:/etc/clickhouse-server/config.d/config.xml
+      - ./resources/click-house/users.xml:/etc/clickhouse-server/users.d/users.xml
+      - ./resources/click-house/schemas/errors.proto:/var/lib/clickhouse/format_schemas/errors.proto
+      - ./resources/click-house/schemas/run_infos.proto:/var/lib/clickhouse/format_schemas/run_infos.proto
+      - ./resources/click-house/schemas/steps.proto:/var/lib/clickhouse/format_schemas/steps.proto
+      - ./resources/click-house/schemas/system_stats.proto:/var/lib/clickhouse/format_schemas/system_stats.proto
+      - ./resources/click-house/schemas/urls.proto:/var/lib/clickhouse/format_schemas/urls.proto
+      - ./resources/click-house/schemas/verifications.proto:/var/lib/clickhouse/format_schemas/verifications.proto
+      - ./resources/click-house/schemas/vu_logs.proto:/var/lib/clickhouse/format_schemas/vu_logs.proto
+      - ./resources/click-house/schemas/vu_statuses.proto:/var/lib/clickhouse/format_schemas/vu_statuses.proto
     ports:
-      - "${BROWSERUP_INFLUX_DB_PORT}:8086"
-    environment:
-      - INFLUXDB_DB=browserup
-      - INFLUX_DB_USER=browserup
-      - INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
-      - INFLUXDB_LOGGING_LEVEL=debug
-      - INFLUXDB_DATA_MAX_SERIES_PER_DATABASE=0
-      - INFLUXDB_DATA_MAX_VALUES_PER_TAG=0
-      - INFLUXDB_HTTP_LOG_ENABLED=false
+      - '8123:8123'
+      - '9000:9000'
     depends_on:
+      - zookeeper
+      - nats
+      - nats-1
       - fluentbit
     logging:
       driver: "fluentd"
       options:
         fluentd-address: "localhost:24224"
 
-  mysql:
-    container_name: browserup_mysql
-    image: mysql:8.0.31-oracle
-    command: mysqld --default-authentication-plugin=mysql_native_password
+  zookeeper:
+    image: 'clickhouse/clickhouse-keeper:24.4'
+    container_name: zookeeper
+    hostname: zookeeper
     volumes:
-      - mysql-volume:/var/lib/mysql
-      - ./resources/init.sql:/docker-entrypoint-initdb.d/z-create-test-db.sql
+      - ./resources/click-house/keeper-config.xml:/etc/clickhouse-keeper/keeper_config.xml
     ports:
-      - "${BROWSERUP_MYSQL_PORT}:3306"
-    environment:
-      - MYSQL_RANDOM_ROOT_PASSWORD=yes
-      - MYSQL_DATABASE=browserup_development
-      - MYSQL_USER=browserup
-      - MYSQL_PASSWORD=${BROWSERUP_MYSQL_PASSWORD}
+      - '9181:9181'
     depends_on:
       - fluentbit
     logging:
       driver: "fluentd"
       options:
         fluentd-address: "localhost:24224"
 
-  redis:
-    container_name: browserup_redis
-    image: redis:6.2.5
+  nats:
+    image: nats
     ports:
-      - "${BROWSERUP_REDIS_PORT}:6379"
-    depends_on:
-      - fluentbit
-    logging:
-      driver: "fluentd"
-      options:
-        fluentd-address: "localhost:24224"
+      - "6222:6222"
+      - "4222:4222"
+      - "8222:8222"
+    volumes:
+      - ./resources/nats/nats-server.conf:/etc/nats/nats-server.conf
+    command: "--cluster_name NATS --cluster nats://0.0.0.0:6222 --http_port 8222 -c /etc/nats/nats-server.conf"
+
+  nats-1:
+    image: nats
+    volumes:
+      - ./resources/nats/nats-server.conf:/etc/nats/nats-server.conf
+    command: "--cluster_name NATS --cluster nats://0.0.0.0:6222 --routes=nats://nats:6222 -c /etc/nats/nats-server.conf"
+    depends_on: ["nats"]
 
   rabbitmq:
     container_name: browserup_rabbitmq
@@ -314,28 +287,6 @@ services:
       options:
         fluentd-address: "localhost:24224"
 
-  zookeeper:
-    container_name: browserup_zookeeper
-    image: zookeeper:3.8.1
-    volumes:
-      - ./resources/zoo.cfg:/conf/zoo.cfg
-      - zookeeper-data-volume:/data
-      - zookeeper-datalog-volume:/datalog
-    ports:
-      - "${BROWSERUP_ZOOKEEPER_CLIENT_PORT}:2181"
-    environment:
-      - ZOO_TICK_TIME=300000  # Helpful for debugging
-      - ZOO_CFG_EXTRA=warn
-      - ZOO_MAX_CLIENT_CNXNS=${BROWSERUP_ZOO_MAX_CLIENT_CNXNS}
-    depends_on:
-      - fluentbit
-      - loki
-      - grafana
-    logging:
-      driver: "fluentd"
-      options:
-        fluentd-address: "localhost:24224"
-
   minio:
     container_name: browserup_minio
     image: quay.io/minio/minio:RELEASE.2022-01-08T03-11-54Z
@@ -374,8 +325,6 @@ services:
   # storage
   loki:
     image: grafana/loki:2.8.6
-    ports:
-      - "24310:3100"
     volumes:
       - ./resources/loki-config.yaml:/etc/loki/local-config.yaml
 
@@ -396,6 +345,18 @@ services:
       - ./resources/grafana-default-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-default-dashboard.json
       - ./resources/grafana-zookeeper-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-zookeeper-dashboard.json
       - ./resources/grafana-grid-services-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-grid-services-dashboard.json
+      - ./resources/grafana-cadvisor-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-cadvisor-dashboard.json
+      - ./resources/grafana-clickhouse-dashboard.json:/etc/grafana/provisioning/dashboards/grafana-clickhouse-dashboard.json
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor
+    container_name: cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /dev/disk:/dev/disk/:ro
 
   prometheus:
     image: prom/prometheus:v2.44.0
@@ -405,8 +366,6 @@ services:
       - "${PROMETHEUS_PORT}:9090"
 
 volumes:
-  influxdb-volume:
-  mysql-volume:
   rabbitmq-volume:
   zookeeper-data-volume:
   zookeeper-datalog-volume:

lib/docker_compose/resources/click-house/config.xml

@@ -0,0 +1,35 @@
+<clickhouse replace="true">
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+    <logger>
+        <console>1</console>
+    </logger>
+    <display_name>ch-1S_1K</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <user_directories>
+        <users_xml>
+            <path>users.xml</path>
+        </users_xml>
+        <local_directory>
+            <path>/tmp/lib/clickhouse/access/</path>
+        </local_directory>
+    </user_directories>
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+    <zookeeper>
+        <node>
+            <host>zookeeper</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+    <prometheus>
+        <endpoint>/metrics</endpoint>
+        <port>9363</port>
+        <metrics>true</metrics>
+        <events>true</events>
+        <asynchronous_metrics>true</asynchronous_metrics>
+        <errors>true</errors>
+    </prometheus>
+</clickhouse>