Merge pull request #20 from browserup/add-trace-spans-to-har

Add trace spans to har that we can grab in grid to use when filing urls, steps.

Author: ericbeland

CLAUDE.md

@@ -2,12 +2,26 @@ This is a customized fork of the mitmproxy project, maintained by a company name
 is used to man-in-the-middle connections to provide debugging info, allow for security research
 and other uses.
 
-Browserup uses the mitmproxy to capture traffic from within containers that are run during a load test.
-This captured traffic, through mitmproxy addons, is used to build a HAR (HTTP Archive) file. The HAR
-file is made available via an API.  The API is also used to control the proxy, and to add custom metrics
-to a har at runtime, as well as adding verifications for HAR content. Clients for 
-this API are generated in multiple languages via the open api specification.  The clients live in /clients
-which should be ignored, as they are generated files.
-
-Browserup extends mitmproxy's mitmdump executable with addons 
-The most important code for browserup live in mitmproxy/addons/browserup
+The Browserup proxy fork uses a variation of the mitmproxy's "mitmdump" (no UI) mode  to capture traffic from within containers that are run during a load test. 
+This captured traffic, captured through mitmproxy addons, is 
+used to build a HAR (HTTP Archive) file. The HAR file is made available via an API. The HAR also gets decorated with custom data, which start with an underscore.
+
+The proxy offers an API that is also used to control the proxy, and to add custom metrics to a har at runtime, as well as adding verifications for HAR 
+content. Clients for this API are generated in multiple languages via the open api specification.  The clients live in /clients which should be ignored, as 
+they are generated files. 
+
+Unusually, the schema .json files are regenerated when the tests are run. The definitions of the schemas are in mitmproxy/addons/browserup/har/har_schemas.py.
+They are defined in marshmallow.
+
+The command to run the tests is: tox -e py
+
+The command to check the linting, which we should do after all changes are done is:
+tox -e lint
+
+
+The BrowserUp proxy extends mitmproxy's mitmdump executable with addons. The most important code for browserup lives in the subfolder mitmproxy/addons/browserup.
+It is implemented almost entirely by changes and additions to addons. It is undesirable to change files outside of this path, and the corresponding test path, as that
+makes it difficult to take merges from the upstream mitmproxy project. 
+
+It is also ok to change items in the subfolder mitmproxy/tools/browserup_proxy.py as new
+addons are added. This is where they are loaded.

browserup-proxy.schema.json

@@ -25,7 +25,7 @@
       "url": "logo.png"
     },
     "title": "BrowserUp MitmProxy",
-    "version": "1.24"
+    "version": "1.25"
   },
   "paths": {
     "/har": {
@@ -387,52 +387,60 @@
         "type": "object",
         "properties": {
           "onContentLoad": {
-            "type": "number",
+            "type": "integer",
             "description": "onContentLoad per the browser"
           },
           "onLoad": {
-            "type": "number",
+            "type": "integer",
             "description": "onLoad per the browser"
           },
           "_firstInputDelay": {
-            "type": "number",
+            "type": "integer",
             "description": "firstInputDelay from the browser"
           },
           "_firstPaint": {
-            "type": "number",
+            "type": "integer",
             "description": "firstPaint from the browser"
           },
           "_cumulativeLayoutShift": {
-            "type": "number",
+            "type": "integer",
             "description": "cumulativeLayoutShift metric from the browser"
           },
           "_largestContentfulPaint": {
-            "type": "number",
+            "type": "integer",
             "description": "largestContentfulPaint from the browser"
           },
           "_domInteractive": {
-            "type": "number",
+            "type": "integer",
             "description": "domInteractive from the browser"
           },
           "_firstContentfulPaint": {
-            "type": "number",
+            "type": "integer",
             "description": "firstContentfulPaint from the browser"
           },
           "_dns": {
-            "type": "number",
+            "type": "integer",
             "description": "dns lookup time from the browser"
           },
           "_ssl": {
-            "type": "number",
+            "type": "integer",
             "description": "Ssl connect time from the browser"
           },
           "_timeToFirstByte": {
-            "type": "number",
+            "type": "integer",
             "description": "Time to first byte of the page's first request per the browser"
           },
           "_href": {
             "type": "string",
             "description": "Top level href, including hashtag, etc per the browser"
+          },
+          "_span_id": {
+            "type": "string",
+            "description": "W3C Trace Context span ID for this page"
+          },
+          "_parent_id": {
+            "type": "string",
+            "description": "W3C Trace Context parent span ID (typically the HAR log span ID)"
           }
         }
       },
@@ -602,7 +610,7 @@
             "description": "Name of Custom Metric to add to the page under _metrics"
           },
           "value": {
-            "type": "number",
+            "type": "integer",
             "format": "double",
             "description": "Value for the metric"
           }
@@ -856,6 +864,14 @@
                   "$ref": "#/components/schemas/HarEntry"
                 }
               },
+              "_trace_id": {
+                "type": "string",
+                "description": "W3C Trace Context trace ID for distributed tracing"
+              },
+              "_span_id": {
+                "type": "string",
+                "description": "W3C Trace Context span ID for this HAR trace root"
+              },
               "comment": {
                 "type": "string"
               }
@@ -1314,6 +1330,18 @@
               "$ref": "#/components/schemas/WebSocketMessage"
             }
           },
+          "_span_id": {
+            "type": "string",
+            "description": "W3C Trace Context span ID for this entry"
+          },
+          "_parent_id": {
+            "type": "string",
+            "description": "W3C Trace Context parent span ID (typically the page span ID)"
+          },
+          "_trace_id": {
+            "type": "string",
+            "description": "W3C Trace Context trace ID for distributed tracing"
+          },
           "connection": {
             "type": "string"
           },
@@ -1367,6 +1395,14 @@
             },
             "default": []
           },
+          "_span_id": {
+            "type": "string",
+            "description": "W3C Trace Context span ID for this page"
+          },
+          "_parent_id": {
+            "type": "string",
+            "description": "W3C Trace Context parent span ID (typically the HAR log span ID)"
+          },
           "pageTimings": {
             "$ref": "#/components/schemas/PageTimings"
           },

clients/build-clients.sh

@@ -1,38 +1,53 @@
 #!/bin/bash
 
-# requires openapi-generator installed
+# requires openapi-generator installed and accessible in your PATH
 
 DIR="$(dirname "${BASH_SOURCE[0]}")"
-
-# we need to regenerate the schema in the top level dir
-
-rm -rf markdown && npx openapi-generator-cli generate \
--g markdown -i "${DIR}/../browserup-proxy.schema.json" \
--o markdown
-
-rm -rf csharp && npx openapi-generator-cli generate \
---package-name BrowserUpMitmProxyClient \
--g csharp-netcore -i "${DIR}/../browserup-proxy.schema.json" \
--o csharp -c config-csharp.yaml
-
-rm -rf java && npx openapi-generator-cli generate \
---package-name BrowserUpMitmProxyClient \
--g java -i "${DIR}/../browserup-proxy.schema.json" \
--o java -c config-java.yaml
-
-rm -rf javascript && npx openapi-generator-cli generate \
---package-name BrowserUpMitmProxyClient \
--g javascript -i "${DIR}/../browserup-proxy.schema.json" \
--o javascript -c config-javascript.yaml
-
-rm -rf python && npx openapi-generator-cli generate \
---package-name BrowserUpMitmProxyClient \
--g python -i "${DIR}/../browserup-proxy.schema.json" \
--o python -c config-python.yaml
-
-rm -rf ruby && npx openapi-generator-cli generate \
---package-name BrowserUpMitmProxyClient \
--g ruby -i "${DIR}/../browserup-proxy.schema.json" \
--o ruby -c config-ruby.yaml
+SCHEMA="${DIR}/../browserup-proxy.schema.json"
+
+# Generate Markdown documentation
+rm -rf markdown && openapi-generator generate \
+    -g markdown -i "${SCHEMA}" \
+    -o markdown
+
+# Generate C# (.NET Core) client
+rm -rf csharp && openapi-generator generate \
+    --package-name BrowserUpMitmProxyClient \
+    -g csharp -i "${SCHEMA}" \
+    -o csharp -c config-csharp.yaml
+
+# Generate Java client
+rm -rf java && openapi-generator generate \
+    --package-name BrowserUpMitmProxyClient \
+    -g java -i "${SCHEMA}" \
+    -o java -c config-java.yaml
+
+# Generate JavaScript client
+rm -rf javascript && openapi-generator generate \
+    --package-name BrowserUpMitmProxyClient \
+    -g javascript -i "${SCHEMA}" \
+    -o javascript -c config-javascript.yaml
+
+# Generate Python client
+rm -rf python && openapi-generator generate \
+    --package-name BrowserUpMitmProxyClient \
+    -g python -i "${SCHEMA}" \
+    -o python -c config-python.yaml
+
+# Generate Ruby client
+rm -rf ruby && openapi-generator generate \
+    --package-name BrowserUpMitmProxyClient \
+    -g ruby -i "${SCHEMA}" \
+    -o ruby -c config-ruby.yaml
+
+rm -rf ruby && openapi-generator generate \
+    --package-name BrowserUpMitmProxyClient \
+    -g ruby -i "${SCHEMA}" \
+    -o ruby -c config-ruby.yaml
+
+rm -rf go && openapi-generator generate \
+    --package-name BrowserUpMitmProxyClient \
+    -g go -i "${SCHEMA}" \
+    -o go -c config-go.yaml
 
 ./post-build-java-client.sh

clients/config-go.yaml

@@ -0,0 +1,14 @@
+# Config file for OpenAPI Generator Go client
+
+# The package name for the generated code.
+# In Go, package names should be all lower case without special characters.
+packageName: browserupmitmproxyclient
+
+# (Optional) Module name for the Go module (go.mod).
+moduleName: BrowserUpMitmProxyClient
+
+# Project metadata
+projectName: browserup-mitmproxy-client
+projectVersion: 1.0.0
+projectDescription: "REST client for the BrowserUp Proxy"
+licenseName: MIT

clients/config-python.yaml

@@ -6,7 +6,8 @@ infoName: BrowserUp, Inc.
 packageVersion: "1.0.1"
 
 packageUrl: https://github.com/browserup/mitmproxy
-
+  
+enablePostProcessFile: true
 
 templateDir: python-overlay
 files: