Update for new observer service

Author: kiturutin

lib/commands/stop.mjs

@@ -77,8 +77,8 @@ export async function sendStopRunRequest(runId, apiToken) {
     try {
         log.debug(`Sending stop Run with id: ${runId} request`);
 
-        await WebConsoleClient.sendPathRequest({
-            path: `load/runs/${runId}`,
+        await WebConsoleClient.sendGetRequest({
+            path: `load/runs/${runId}/stop`,
             qParams: {
                 api_token: apiToken
             }

lib/config/env.properties

@@ -42,6 +42,8 @@ BROWSERUP_INFLUX_TEST_DB_HOST=influx-test-db
 BROWSERUP_INFLUX_TEST_DB_PORT=23011
 BROWSERUP_INFLUX_DB_PASSWORD=just_testing
 
+BROWSERUP_LOKI_HOST=loki
+
 BROWSERUP_S3_MINIO_HOST_PORT_1=23012
 
 BROWSERUP_GRAFANA_PORT=23013

lib/docker_compose/docker-compose.yml

@@ -47,7 +47,6 @@ services:
     image: docker.io/browserup/grid-java-api:latest
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
-      - //wsl$/docker-desktop-data:/var/lib/docker/containers
     ports:
       - "${BROWSERUP_GRID_JAVA_API_PORT}:8080"
     restart: on-failure
@@ -95,9 +94,9 @@ services:
       - rabbitmq
       - fluentbit
     logging:
-      driver: "json-file"
+      driver: "fluentd"
       options:
-        tag: "{{.Name}}"
+        fluentd-address: "localhost:24224"
 
   grid-java-coordinator:
     container_name: browserup_grid-java-coordinator
@@ -144,6 +143,34 @@ services:
       - rabbitmq
       - grid-java-api
       - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
+
+  grid-java-observer:
+    container_name: browserup_grid-java-observer
+    image: docker.io/browserup/observer:latest
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: on-failure
+    environment:
+      - BROWSERUP_RABBITMQ_HOST=${BROWSERUP_RABBITMQ_HOST}
+      - BROWSERUP_RABBITMQ_DEFAULT_USER=browserup
+      - BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
+      - BROWSERUP_RABBITMQ_QUEUE_PORT=5672
+      - BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
+      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=2181
+      - BROWSERUP_IS_CLOUD=false
+    depends_on:
+      - zookeeper
+      - rabbitmq
+      - grid-java-api
+      - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   webconsole:
     container_name: browserup_webconsole
@@ -168,6 +195,7 @@ services:
       - BROWSERUP_INFLUX_DB_USER=browserup
       - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
       - BROWSERUP_GRID_SERVICE_URL=http://${BROWSERUP_GRID_JAVA_API_HOST}:8080/grid
+      - BROWSERUP_LOKI_SERVICE_URL=http://${BROWSERUP_LOKI_HOST}:3100
       - BROWSERUP_IS_CLOUD=false
       - BROWSERUP_REDIS_URL=redis://${BROWSERUP_REDIS_HOST}:6379/1
       - BROWSERUP_S3_MINIO_ACCESS_KEY_ID=${BROWSERUP_S3_MINIO_ACCESS_KEY_ID}
@@ -192,6 +220,10 @@ services:
       - fluentbit
     volumes:
       - ./containers:/var/lib/containers
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   chronograf:
     image: chronograf:1.7.16
@@ -200,6 +232,10 @@ services:
     depends_on:
       - influxdb
       - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   influxdb:
     container_name: browserup_influxdb
@@ -218,6 +254,10 @@ services:
       - INFLUXDB_HTTP_LOG_ENABLED=false
     depends_on:
       - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   mysql:
     container_name: browserup_mysql
@@ -235,6 +275,10 @@ services:
       - MYSQL_PASSWORD=${BROWSERUP_MYSQL_PASSWORD}
     depends_on:
       - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   redis:
     container_name: browserup_redis
@@ -243,6 +287,10 @@ services:
       - "${BROWSERUP_REDIS_PORT}:6379"
     depends_on:
       - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   rabbitmq:
     container_name: browserup_rabbitmq
@@ -263,6 +311,10 @@ services:
       retries: 3
     depends_on:
       - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   zookeeper:
     container_name: browserup_zookeeper
@@ -281,6 +333,10 @@ services:
       - fluentbit
       - loki
       - grafana
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
 
   minio:
     container_name: browserup_minio
@@ -305,22 +361,24 @@ services:
 
   # router
   fluentbit:
-    image: fluent/fluent-bit:2.2.2
+    image: fluent/fluent-bit:3.0.3-debug
     depends_on:
       - loki
     environment:
       - LOG_PATH=/tmp
     ports:
       - "24224:24224"
       - "24224:24224/udp"
-      - "2020:2020"
+      - "24020:2020"
+      - "24888:8888"
     volumes:
-      - ./resources/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
-      - ./resources/parsers.conf:/fluent-bit/etc/parsers.conf
+      - ./resources/fluent-bit:/fluent-bit/etc
 
   # storage
   loki:
     image: grafana/loki:2.8.6
+    ports:
+      - "24310:3100"
     volumes:
       - ./resources/loki-config.yaml:/etc/loki/local-config.yaml
 

lib/docker_compose/resources/fluent-bit.conf

@@ -0,0 +1,31 @@
+[SERVICE]
+    HTTP_Server  On
+    HTTP_Listen  0.0.0.0
+    HTTP_PORT    2020
+    Hot_Reload   On
+    Parsers_File  /fluent-bit/etc/parsers.conf
+
+[FILTER]
+    Name modify
+    Match user_container_log
+    Condition Key_value_matches log ^browserup_user_log.*
+    Add log_type user
+
+[FILTER]
+    Name modify
+    Match user_container_log
+    Condition Key_Value_Does_Not_Match log ^browserup_user_log.*
+    Add log_type system
+
+[INPUT]
+    name http
+    listen 0.0.0.0
+    port 8888
+
+[Output]
+    name loki
+    match *
+    host loki
+    port 3100
+    labels container_name=$container_name $log_type
+    line_format json

lib/docker_compose/resources/fluent-bit/fluent-bit.conf

@@ -327,6 +327,8 @@ Resources:
               Value: !Ref ClusterVpcId
             - Name: BROWSERUP_CLUSTER_NAME
               Value: !Ref ClusterName
+            - Name: BROWSERUP_OBSERVER_VERSION
+              Value: !Ref ServicesImageTag
             - Name: EC2_INSTANCE_FAMILY
               Value: c5
             - Name: BROWSERUP_INFLUX_DB_HOST
@@ -359,6 +361,10 @@ Resources:
               Value: 9000
             - Name: BROWSERUP_S3_MINIO_PORT_2
               Value: 9001
+            - Name: BROWSERUP_LOKI_HOST
+              Value: !Join ['', ['loki.', !Ref PrivateDnsNamespaceName]]
+            - Name: BROWSERUP_LOKI_PORT
+              Value: 3100
             - Name: BROWSERUP_S3_MINIO_ACCESS_KEY_ID
               Value: !Ref S3MinioAccessKeyID
             - Name: BROWSERUP_S3_MINIO_SECRET_ACCESS_KEY

lib/docker_compose/resources/parsers.conf

@@ -43,7 +43,7 @@ export class LocalClusterPilot extends ClusterPilot {
         if (process.env.CHRONOGRAF) backendServices.push("chrono");
         const frontendServices = [];
         if (!process.env.DEBUG_WEBCONSOLE) frontendServices.push("webconsole");
-        const middlewareServices = ["grid-java-coordinator", "grid-java-api"];
+        const middlewareServices = ["grid-java-coordinator", "grid-java-api", "grid-java-observer"];
         const allServices = backendServices.concat(middlewareServices, frontendServices);
 
         const envs = {