Logging update

lib/commands/stop.mjs

@@ -77,8 +77,8 @@ export async function sendStopRunRequest(runId, apiToken) {
     try {
         log.debug(`Sending stop Run with id: ${runId} request`);
 
-        await WebConsoleClient.sendPathRequest({
-            path: `load/runs/${runId}`,
+        await WebConsoleClient.sendGetRequest({
+            path: `load/runs/${runId}/stop`,
             qParams: {
                 api_token: apiToken
             }

lib/config/env.properties

@@ -42,6 +42,8 @@ BROWSERUP_INFLUX_TEST_DB_HOST=influx-test-db
 BROWSERUP_INFLUX_TEST_DB_PORT=23011
 BROWSERUP_INFLUX_DB_PASSWORD=just_testing
 
+BROWSERUP_LOKI_HOST=loki
+
 BROWSERUP_S3_MINIO_HOST_PORT_1=23012
 
 BROWSERUP_GRAFANA_PORT=23013

lib/docker_compose/docker-compose.yml

@@ -148,6 +148,30 @@ services:
       options:
         fluentd-address: "localhost:24224"
 
+  grid-java-observer:
+    container_name: browserup_grid-java-observer
+    image: docker.io/browserup/observer:latest
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: on-failure
+    environment:
+      - BROWSERUP_RABBITMQ_HOST=${BROWSERUP_RABBITMQ_HOST}
+      - BROWSERUP_RABBITMQ_DEFAULT_USER=browserup
+      - BROWSERUP_RABBITMQ_DEFAULT_PASS=${BROWSERUP_RABBITMQ_DEFAULT_PASS}
+      - BROWSERUP_RABBITMQ_QUEUE_PORT=5672
+      - BROWSERUP_ZOOKEEPER_HOST=${BROWSERUP_ZOOKEEPER_HOST}
+      - BROWSERUP_ZOOKEEPER_CLIENT_PORT=2181
+      - BROWSERUP_IS_CLOUD=false
+    depends_on:
+      - zookeeper
+      - rabbitmq
+      - grid-java-api
+      - fluentbit
+    logging:
+      driver: "fluentd"
+      options:
+        fluentd-address: "localhost:24224"
+
   webconsole:
     container_name: browserup_webconsole
     image: docker.io/browserup/webconsole:latest
@@ -171,6 +195,7 @@ services:
       - BROWSERUP_INFLUX_DB_USER=browserup
       - BROWSERUP_INFLUX_DB_PASSWORD=${BROWSERUP_INFLUX_DB_PASSWORD}
       - BROWSERUP_GRID_SERVICE_URL=http://${BROWSERUP_GRID_JAVA_API_HOST}:8080/grid
+      - BROWSERUP_LOKI_SERVICE_URL=http://${BROWSERUP_LOKI_HOST}:3100
       - BROWSERUP_IS_CLOUD=false
       - BROWSERUP_REDIS_URL=redis://${BROWSERUP_REDIS_HOST}:6379/1
       - BROWSERUP_S3_MINIO_ACCESS_KEY_ID=${BROWSERUP_S3_MINIO_ACCESS_KEY_ID}
@@ -197,8 +222,6 @@ services:
       driver: "fluentd"
       options:
         fluentd-address: "localhost:24224"
-    volumes:
-      - ./containers:/var/lib/containers
 
   chronograf:
     image: chronograf:1.7.16
@@ -336,21 +359,23 @@ services:
 
   # router
   fluentbit:
-    image: fluent/fluent-bit:2.1.10
+    image: fluent/fluent-bit:3.0.3
     depends_on:
       - loki
     environment:
       - LOG_PATH=/tmp
     ports:
-      - "24224:24224"
-      - "24224:24224/udp"
-      - "2020:2020"
+      - "24020:2020"
+      - "24224:8889"
+      - "24888:8888"
     volumes:
-      - ./resources/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
+      - ./resources/fluent-bit:/fluent-bit/etc
 
   # storage
   loki:
     image: grafana/loki:2.8.6
+    ports:
+      - "24310:3100"
     volumes:
       - ./resources/loki-config.yaml:/etc/loki/local-config.yaml
 

lib/docker_compose/resources/fluent-bit/fluent-bit.conf

@@ -0,0 +1,36 @@
+[SERVICE]
+    HTTP_Server  On
+    HTTP_Listen  0.0.0.0
+    HTTP_PORT    2020
+    Hot_Reload   On
+    Parsers_File  /fluent-bit/etc/parsers.conf
+
+[FILTER]
+    Name modify
+    Match user_container_log
+    Condition Key_value_matches log ^browserup_user_log.*
+    Add log_type user
+
+[FILTER]
+    Name modify
+    Match user_container_log
+    Condition Key_Value_Does_Not_Match log ^browserup_user_log.*
+    Add log_type system
+
+[INPUT]
+    name http
+    listen 0.0.0.0
+    port 8888
+
+[INPUT]
+    Name        forward
+    Listen      0.0.0.0
+    Port        8889
+
+[Output]
+    name loki
+    match *
+    host loki
+    port 3100
+    labels container_name=$container_name $log_type
+    line_format json

lib/services/aws/aws-resources/clusterServiceStackTemplate.yaml

@@ -327,6 +327,8 @@ Resources:
               Value: !Ref ClusterVpcId
             - Name: BROWSERUP_CLUSTER_NAME
               Value: !Ref ClusterName
+            - Name: BROWSERUP_OBSERVER_VERSION
+              Value: !Ref ServicesImageTag
             - Name: EC2_INSTANCE_FAMILY
               Value: c5
             - Name: BROWSERUP_INFLUX_DB_HOST
@@ -359,6 +361,10 @@ Resources:
               Value: 9000
             - Name: BROWSERUP_S3_MINIO_PORT_2
               Value: 9001
+            - Name: BROWSERUP_LOKI_HOST
+              Value: !Join ['', ['loki.', !Ref PrivateDnsNamespaceName]]
+            - Name: BROWSERUP_LOKI_PORT
+              Value: 3100
             - Name: BROWSERUP_S3_MINIO_ACCESS_KEY_ID
               Value: !Ref S3MinioAccessKeyID
             - Name: BROWSERUP_S3_MINIO_SECRET_ACCESS_KEY

lib/services/docker_client.mjs

@@ -319,14 +319,14 @@ export class DockerClient {
             return;
         }
 
-        const cmd = `docker stop ${minionsContainersIds.join(' ')};`;
+        const cmd = `docker stop ${minionsContainersIds.join(' ')}`;
         try {
             await this.execCommand(cmd);
         } catch (e) {
             throw decoratedError({msg: `Failed to stop minion containers`, error: e});
         }
 
-        const cmd2 = `docker rm ${minionsContainersIds.join(' ')};`;
+        const cmd2 = `docker rm ${minionsContainersIds.join(' ')}`;
         try {
             await this.execCommand(cmd2);
         } catch (errors) {

lib/services/local_cluster_pilot.mjs

@@ -43,7 +43,7 @@ export class LocalClusterPilot extends ClusterPilot {
         if (process.env.CHRONOGRAF) backendServices.push("chrono");
         const frontendServices = [];
         if (!process.env.DEBUG_WEBCONSOLE) frontendServices.push("webconsole");
-        const middlewareServices = ["grid-java-coordinator", "grid-java-api"];
+        const middlewareServices = ["grid-java-coordinator", "grid-java-api", "grid-java-observer"];
         const allServices = backendServices.concat(middlewareServices, frontendServices);
 
         const envs = {

lib/utils/docker.mjs

@@ -92,7 +92,7 @@ export function copyFilesToDotFolderIfMissing(servicesImageTag) {
 
   servicesImageTag = servicesImageTag || SERVICES_VERSION;
 
-  DcConfigEditor.updateServicesImageTags(dcYmlPath, ["grid-java-api", "grid-java-coordinator", "webconsole"], servicesImageTag);
+  DcConfigEditor.updateServicesImageTags(dcYmlPath, ["grid-java-api", "grid-java-coordinator", "webconsole", "grid-java-observer"], servicesImageTag);
 
   DcConfigEditor.updateEnvForService(
     dcYmlPath,