tp: speed up JSON tokenization/parsing by ~9x

BUILD

@@ -2211,6 +2211,7 @@ perfetto_filegroup(
 perfetto_filegroup(
     name = "src_trace_processor_importers_json_minimal",
     srcs = [
+        "src/trace_processor/importers/json/json_parser.h",
         "src/trace_processor/importers/json/json_utils.cc",
         "src/trace_processor/importers/json/json_utils.h",
     ],

CHANGELOG

@@ -4,9 +4,10 @@ Unreleased:
   SQL Standard library:
     *
   Trace Processor:
-    *
+    * Improved performance of parsing JSON by 7x. JSON traces should now load
+      much faster in the Perfetto UI and the trace processor.
   UI:
-    *
+    * Improved JSON loading performance. See `Trace Processor` section above.
   SDK:
     *
 

src/trace_processor/importers/common/BUILD.gn

@@ -107,7 +107,10 @@ source_set("common") {
 
 source_set("trace_parser_hdr") {
   sources = [ "trace_parser.h" ]
-  deps = [ "../../../../gn:default_deps" ]
+  deps = [
+    ":parser_types",
+    "../../../../gn:default_deps",
+  ]
 }
 
 source_set("parser_types") {

src/trace_processor/importers/common/parser_types.h

@@ -20,13 +20,16 @@
 #include <array>
 #include <cstdint>
 #include <functional>
+#include <limits>
+#include <memory>
 #include <optional>
-#include <string>
 #include <utility>
 #include <variant>
+#include <vector>
 
 #include "perfetto/trace_processor/ref_counted.h"
 #include "perfetto/trace_processor/trace_blob_view.h"
+#include "src/trace_processor/containers/interval_tree.h"
 #include "src/trace_processor/containers/string_pool.h"
 #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h"
 
@@ -57,16 +60,72 @@ struct alignas(8) InlineSchedWaking {
 static_assert(sizeof(InlineSchedWaking) == 16);
 
 struct alignas(8) JsonEvent {
-  struct Begin {};
-  struct End {};
-  struct Scoped {
-    int64_t dur;
+  int64_t dur = std::numeric_limits<int64_t>::max();
+
+  uint32_t pid = 0;
+  uint32_t tid = 0;
+
+  StringPool::Id name = StringPool::Id::Null();
+  StringPool::Id cat = StringPool::Id::Null();
+
+  union IdStrOrUint64 {
+    StringPool::Id id_str;
+    uint64_t id_uint64;
   };
-  struct Other {};
-  using Type = std::variant<Begin, End, Scoped, Other>;
+  IdStrOrUint64 id;
+  IdStrOrUint64 bind_id;
+
+  int64_t tts = std::numeric_limits<int64_t>::max();
+  int64_t tdur = std::numeric_limits<int64_t>::max();
+  int64_t async_cookie = std::numeric_limits<int64_t>::max();
+
+  std::unique_ptr<char[]> args;
+
+  char phase = '\0';
+
+  uint64_t flow_in : 1;
+  uint64_t flow_out : 1;
 
-  std::string value;
-  Type type;
+  uint64_t pid_is_string_id : 1;
+  uint64_t tid_is_string_id : 1;
+
+  uint64_t bind_enclosing_slice : 1;
+
+  enum class IdType : uint64_t {
+    kNone = 0,
+    kString = 1,
+    kUint64 = 2,
+  };
+  IdType id_type : 2;
+  IdType bind_id_type : 2;
+
+  enum class Scope : uint64_t {
+    kNone = 0,
+    kGlobal = 1,
+    kProcess = 2,
+    kThread = 3,
+  };
+  Scope scope : 2;
+
+  enum class AsyncCookieType : uint64_t {
+    kId,
+    kId2Local,
+    kId2Global,
+  };
+  AsyncCookieType async_cookie_type : 2;
+
+  uint64_t args_size : 43;
+
+  JsonEvent()
+      : flow_in(false),
+        flow_out(false),
+        pid_is_string_id(false),
+        tid_is_string_id(false),
+        bind_enclosing_slice(false),
+        id_type(IdType::kNone),
+        bind_id_type(IdType::kNone),
+        scope(Scope::kNone),
+        args_size(0) {}
 };
 static_assert(sizeof(JsonEvent) % 8 == 0);
 

src/trace_processor/importers/common/trace_parser.h

@@ -19,6 +19,7 @@
 
 #include <cstdint>
 #include <string>
+#include "src/trace_processor/importers/common/parser_types.h"
 
 namespace perfetto::trace_processor {
 namespace perf_importer {
@@ -63,7 +64,7 @@ class ProtoTraceParser {
 class JsonTraceParser {
  public:
   virtual ~JsonTraceParser();
-  virtual void ParseJsonPacket(int64_t, std::string) = 0;
+  virtual void ParseJsonPacket(int64_t, JsonEvent) = 0;
   virtual void ParseSystraceLine(int64_t, SystraceLine) = 0;
   virtual void ParseLegacyV8ProfileEvent(int64_t, LegacyV8CpuProfileEvent) = 0;
 };

src/trace_processor/importers/json/BUILD.gn

@@ -17,12 +17,15 @@ import("../../../../gn/test.gni")
 
 source_set("minimal") {
   sources = [
+    "json_parser.h",
     "json_utils.cc",
     "json_utils.h",
   ]
   deps = [
     "../../../../gn:default_deps",
+    "../../containers",
     "../../storage",
+    "../../types",
     "../common",
   ]
   if (enable_perfetto_trace_processor_json) {
@@ -41,6 +44,7 @@ if (enable_perfetto_trace_processor_json) {
     deps = [
       ":minimal",
       "../../../../gn:default_deps",
+      "../../containers",
       "../../sorter",
       "../../storage",
       "../../tables",