diff --git a/BUILD b/BUILD index 49d921cbfa5..0286ac41051 100644 --- a/BUILD +++ b/BUILD @@ -2211,6 +2211,7 @@ perfetto_filegroup( perfetto_filegroup( name = "src_trace_processor_importers_json_minimal", srcs = [ + "src/trace_processor/importers/json/json_parser.h", "src/trace_processor/importers/json/json_utils.cc", "src/trace_processor/importers/json/json_utils.h", ], diff --git a/CHANGELOG b/CHANGELOG index a6c5505dc5e..1a798d6e269 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,9 +4,10 @@ Unreleased: SQL Standard library: * Trace Processor: - * + * Improved performance of parsing JSON by 7x. JSON traces should now load + much faster in the Perfetto UI and the trace processor. UI: - * + * Improved JSON loading performance. See `Trace Processor` section above. SDK: * diff --git a/src/trace_processor/importers/common/BUILD.gn b/src/trace_processor/importers/common/BUILD.gn index 95f2d889f30..e229104fe83 100644 --- a/src/trace_processor/importers/common/BUILD.gn +++ b/src/trace_processor/importers/common/BUILD.gn @@ -107,7 +107,10 @@ source_set("common") { source_set("trace_parser_hdr") { sources = [ "trace_parser.h" ] - deps = [ "../../../../gn:default_deps" ] + deps = [ + ":parser_types", + "../../../../gn:default_deps", + ] } source_set("parser_types") { diff --git a/src/trace_processor/importers/common/parser_types.h b/src/trace_processor/importers/common/parser_types.h index 07eb8dfc9b5..8e2feefdd4b 100644 --- a/src/trace_processor/importers/common/parser_types.h +++ b/src/trace_processor/importers/common/parser_types.h @@ -20,13 +20,16 @@ #include #include #include +#include +#include #include -#include #include #include +#include #include "perfetto/trace_processor/ref_counted.h" #include "perfetto/trace_processor/trace_blob_view.h" +#include "src/trace_processor/containers/interval_tree.h" #include "src/trace_processor/containers/string_pool.h" #include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" @@ -57,16 +60,72 @@ struct alignas(8) InlineSchedWaking { static_assert(sizeof(InlineSchedWaking) == 16); struct alignas(8) JsonEvent { - struct Begin {}; - struct End {}; - struct Scoped { - int64_t dur; + int64_t dur = std::numeric_limits::max(); + + uint32_t pid = 0; + uint32_t tid = 0; + + StringPool::Id name = StringPool::Id::Null(); + StringPool::Id cat = StringPool::Id::Null(); + + union IdStrOrUint64 { + StringPool::Id id_str; + uint64_t id_uint64; }; - struct Other {}; - using Type = std::variant; + IdStrOrUint64 id; + IdStrOrUint64 bind_id; + + int64_t tts = std::numeric_limits::max(); + int64_t tdur = std::numeric_limits::max(); + int64_t async_cookie = std::numeric_limits::max(); + + std::unique_ptr args; + + char phase = '\0'; + + uint64_t flow_in : 1; + uint64_t flow_out : 1; - std::string value; - Type type; + uint64_t pid_is_string_id : 1; + uint64_t tid_is_string_id : 1; + + uint64_t bind_enclosing_slice : 1; + + enum class IdType : uint64_t { + kNone = 0, + kString = 1, + kUint64 = 2, + }; + IdType id_type : 2; + IdType bind_id_type : 2; + + enum class Scope : uint64_t { + kNone = 0, + kGlobal = 1, + kProcess = 2, + kThread = 3, + }; + Scope scope : 2; + + enum class AsyncCookieType : uint64_t { + kId, + kId2Local, + kId2Global, + }; + AsyncCookieType async_cookie_type : 2; + + uint64_t args_size : 43; + + JsonEvent() + : flow_in(false), + flow_out(false), + pid_is_string_id(false), + tid_is_string_id(false), + bind_enclosing_slice(false), + id_type(IdType::kNone), + bind_id_type(IdType::kNone), + scope(Scope::kNone), + args_size(0) {} }; static_assert(sizeof(JsonEvent) % 8 == 0); diff --git a/src/trace_processor/importers/common/trace_parser.h b/src/trace_processor/importers/common/trace_parser.h index 8e0f31b33ec..6bf173ce5c1 100644 --- a/src/trace_processor/importers/common/trace_parser.h +++ b/src/trace_processor/importers/common/trace_parser.h @@ -19,6 +19,7 @@ #include #include +#include "src/trace_processor/importers/common/parser_types.h" namespace perfetto::trace_processor { namespace perf_importer { @@ -63,7 +64,7 @@ class ProtoTraceParser { class JsonTraceParser { public: virtual ~JsonTraceParser(); - virtual void ParseJsonPacket(int64_t, std::string) = 0; + virtual void ParseJsonPacket(int64_t, JsonEvent) = 0; virtual void ParseSystraceLine(int64_t, SystraceLine) = 0; virtual void ParseLegacyV8ProfileEvent(int64_t, LegacyV8CpuProfileEvent) = 0; }; diff --git a/src/trace_processor/importers/json/BUILD.gn b/src/trace_processor/importers/json/BUILD.gn index c7921d22655..bc6d2375474 100644 --- a/src/trace_processor/importers/json/BUILD.gn +++ b/src/trace_processor/importers/json/BUILD.gn @@ -17,12 +17,15 @@ import("../../../../gn/test.gni") source_set("minimal") { sources = [ + "json_parser.h", "json_utils.cc", "json_utils.h", ] deps = [ "../../../../gn:default_deps", + "../../containers", "../../storage", + "../../types", "../common", ] if (enable_perfetto_trace_processor_json) { @@ -41,6 +44,7 @@ if (enable_perfetto_trace_processor_json) { deps = [ ":minimal", "../../../../gn:default_deps", + "../../containers", "../../sorter", "../../storage", "../../tables", diff --git a/src/trace_processor/importers/json/json_parser.h b/src/trace_processor/importers/json/json_parser.h new file mode 100644 index 00000000000..9fce388d9b6 --- /dev/null +++ b/src/trace_processor/importers/json/json_parser.h @@ -0,0 +1,878 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_JSON_JSON_PARSER_H_ +#define SRC_TRACE_PROCESSOR_IMPORTERS_JSON_JSON_PARSER_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "perfetto/base/compiler.h" +#include "perfetto/base/logging.h" +#include "perfetto/base/status.h" +#include "perfetto/public/compiler.h" + +namespace perfetto::trace_processor::json { + +// Represents a JSON null value. +struct Null {}; +// Represents a JSON object, holding its raw string content. +struct Object { + std::string_view contents; +}; +// Represents a JSON array, holding its raw string content. +struct Array { + std::string_view contents; +}; +// A variant type representing any valid JSON value. +using JsonValue = + std::variant; + +namespace internal { + +// Internal return codes for parsing functions. +enum class ReturnCode : uint8_t { + kOk, + kError, + kIncompleteInput, +}; + +// Advances the |cur| pointer past any JSON whitespace characters. +// Returns false if |end| is reached before any non-whitespace character. +inline bool SkipWhitespace(const char*& cur, const char* end) { + while (cur != end && + (*cur == ' ' || *cur == '\t' || *cur == '\n' || *cur == '\r')) { + ++cur; + } + return cur != end; +} + +// Processes escape sequences within a string segment and appends the unescaped +// result to |res|. +// |start| and |end| define the string segment (excluding initial/final quotes). +// Sets |status| on error. +inline ReturnCode UnescapeString(const char* start, + const char* end, + std::string& res, + base::Status& status) { + PERFETTO_DCHECK(start != end); + // Pre-allocate string capacity, assuming most characters are not escaped. + res.reserve(static_cast(end - start)); + for (const char* it = start; it != end; ++it) { + if (*it == '\\') { + ++it; + PERFETTO_DCHECK(it != end); + switch (*it) { + case '"': + res += '"'; + break; + case '\\': + res += '\\'; + break; + case '/': + res += '/'; + break; + case 'b': + res += '\b'; + break; + case 'f': + res += '\f'; + break; + case 'n': + res += '\n'; + break; + case 'r': + res += '\r'; + break; + case 't': + res += '\t'; + break; + case 'u': { + // Ensure 4 hex digits follow. + PERFETTO_DCHECK(it + 4 != end); + uint32_t cp = 0; + // Parse the 4 hex digits into a code point. + for (int j = 0; j < 4; ++j) { + char hex = *++it; + cp <<= 4; + if (hex >= '0' && hex <= '9') { + cp += static_cast(hex - '0'); + } else if (hex >= 'a' && hex <= 'f') { + cp += static_cast(hex - 'a' + 10); + } else if (hex >= 'A' && hex <= 'F') { + cp += static_cast(hex - 'A' + 10); + } else { + status = base::ErrStatus("Invalid escape sequence: \\u%c%c%c%c", + it[-3], it[-2], it[-1], hex); + return ReturnCode::kError; + } + } + // Encode the code point as UTF-8. + if (cp <= 0x7F) { + // 1-byte sequence + res += static_cast(cp); + } else if (cp <= 0x7FF) { + // 2-byte sequence + res += static_cast(0xC0 | (cp >> 6)); + res += static_cast(0x80 | (cp & 0x3F)); + } else if (cp <= 0xFFFF) { + // 3-byte sequence + // Check for surrogate pairs, which are not supported directly. + if (cp >= 0xD800 && cp <= 0xDFFF) { + status = base::ErrStatus( + "Invalid escape sequence: \\u%c%c%c%c (code point %u is " + "reserved for surrogate pairs)", + it[-3], it[-2], it[-1], *it, cp); + return ReturnCode::kError; + } + res += static_cast(0xE0 | (cp >> 12)); + res += static_cast(0x80 | ((cp >> 6) & 0x3F)); + res += static_cast(0x80 | (cp & 0x3F)); + } else { + // Code points > 0xFFFF are not supported by \uXXXX in JSON + // (they require surrogate pairs). + status = base::ErrStatus( + "Invalid escape sequence: \\u%c%c%c%c (code point %u > 0xFFFF)", + it[-3], it[-2], it[-1], *it, cp); + return ReturnCode::kError; + } + break; + } + default: + // As per JSON spec, other escaped characters are themselves. + // However, strict parsers might error here. This one is lenient. + // res += *it; // This line was effectively a no-op as it was inside + // `default: break;` + break; + } + } else { + res += *it; + } + } + return ReturnCode::kOk; +} + +// Scans a JSON string from |start| to |end|, updating |out| to point after the +// closing quote. +// |str| will view the content of the string (without quotes). +// |has_escapes| is set if escape sequences are present. +// Sets |err| on parsing errors. +inline ReturnCode ScanString(const char* start, + const char* end, + const char*& out, + std::string_view& str, + bool& has_escapes, + base::Status& err) { + const char* cur = start; + PERFETTO_DCHECK(cur != end); + // Expect a string to start with a double quote. + if (PERFETTO_UNLIKELY(*cur != '"')) { + err = base::ErrStatus("Expected '\"' at the start of string. Got '%c'", + *start); + return ReturnCode::kError; + } + // Start searching for the closing quote from the character after the opening + // quote. + for (const char* str_start = ++cur;;) { + // Find the next double quote. + cur = static_cast( + memchr(cur, '"', static_cast(end - cur))); + // If no quote is found, the input is incomplete. + if (PERFETTO_UNLIKELY(!cur)) { + return ReturnCode::kIncompleteInput; + } + // If the character before the quote is not a backslash, it's the end of the + // string. + if (PERFETTO_LIKELY(cur[-1] != '\\')) { + str = std::string_view(str_start, static_cast(cur - str_start)); + out = cur + 1; + return ReturnCode::kOk; + } + // If it's an escaped quote, skip it and continue searching. + ++cur; + has_escapes = true; + } +} + +// Parses a JSON string, handling escape sequences if necessary. +// |start| and |end| define the input buffer. |out| is updated to point after +// the string. |str| receives the string_view of the parsed string (potentially +// unescaped). |unescaped_str| is used as a buffer if unescaping is needed. Sets +// |status| on error. +inline ReturnCode ParseString(const char* start, + const char* end, + const char*& out, + std::string_view& str, + std::string& unescaped_str, + base::Status& status) { + const char* cur = start; + PERFETTO_DCHECK(start != end); + + bool key_has_escapes = false; + // First, scan the string to identify its boundaries and check for escapes. + if (auto e = ScanString(cur, end, cur, str, key_has_escapes, status); + e != ReturnCode::kOk) { + return e; + } + // If escape sequences were found, unescape the string. + if (PERFETTO_UNLIKELY(key_has_escapes)) { + unescaped_str.clear(); // Clear previous unescaped content. + if (auto e = internal::UnescapeString(str.data(), str.data() + str.size(), + unescaped_str, status); + e != ReturnCode::kOk) { + return e; + } + // Update |str| to point to the unescaped version. + str = unescaped_str; + } + out = cur; + return ReturnCode::kOk; +} + +// Scans input from |start| to |end| to find the end of a block delimited by +// |open_delim| and |close_delim| (e.g., '{' and '}'). +// Handles nested delimiters and strings correctly. +// |out| is updated to point after the |close_delim|. +// Sets |status| on error. +inline ReturnCode ScanToEndOfDelimitedBlock(const char* start, + const char* end, + char open_delim, + char close_delim, + const char*& out, + base::Status& status) { + PERFETTO_DCHECK(start != end); + PERFETTO_DCHECK(*start == open_delim); + + // Start scanning after the opening delimiter. + const char* cur = start + 1; + // Balance of open/close delimiters. + uint32_t bal = 1; + // Dummy for ScanString. + std::string_view sv; + bool has_escapes; + while (cur != end) { + char c = *cur; + if (c == '"') { + // If a string starts, scan past it. + if (auto e = ScanString(cur, end, cur, sv, has_escapes, status); + e != ReturnCode::kOk) { + return e; + } + } else if (c == open_delim) { + // Nested opening delimiter. + ++cur; + ++bal; + } else if (c == close_delim) { + // Closing delimiter. + ++cur; + if (PERFETTO_LIKELY(--bal == 0)) { + // If balance is zero, block end is found. + out = cur; + return ReturnCode::kOk; + } + } else { + // Other characters, just advance. + ++cur; + } + } + // Reached end without closing delimiter. + return ReturnCode::kIncompleteInput; +} + +// Converts a string representation of an integer to int64_t. +// |start| and |end| define the string segment for the number. +// |out| stores the parsed integer. +// Sets |status| on overflow or invalid format. +inline ReturnCode StringToInt64(const char* start, + const char* end, + int64_t& out, + base::Status& status) { + const char* cur = start; + PERFETTO_DCHECK(start != end); + + bool negative = false; + if (*cur == '-') { + negative = true; + cur++; + } + // After a potential sign, there must be at least one digit. + PERFETTO_DCHECK(cur != end); + + out = 0; + // Precompute limits for overflow checking. + const int64_t kAbsMaxDiv10 = std::numeric_limits::max() / 10; + const int kAbsMaxMod10 = std::numeric_limits::max() % 10; + for (; cur != end; ++cur) { + // Should only be called with valid digits. + PERFETTO_DCHECK(std::isdigit(*cur)); + int digit = *cur - '0'; + // Check for overflow before multiplication and addition. + if (out > kAbsMaxDiv10 || (out == kAbsMaxDiv10 && digit > kAbsMaxMod10)) { + // Special case for INT64_MIN, which is -(INT64_MAX + 1). + if (negative && out == kAbsMaxDiv10 && digit == kAbsMaxMod10 + 1) { + // This sequence of operations correctly forms INT64_MIN when negated + // later. + } else { + status = base::ErrStatus("Integer overflow parsing '%.*s'", + int(end - start), start); + return ReturnCode::kError; + } + } + out = out * 10 + digit; + } + if (negative) { + out = -out; + } + return ReturnCode::kOk; +} + +// Converts a string representation of a floating-point number to double. +// |start| and |end| define the string segment for the number. +// |out| stores the parsed double. +// Sets |status| on overflow, underflow (to 0.0), NaN, or invalid format. +inline ReturnCode StringToDouble(const char* start, + const char* end, + double& out, + base::Status& status) { + const char* cur = start; + PERFETTO_DCHECK(cur != end); + + bool negative = false; + if (*cur == '-') { + negative = true; + ++cur; + } + PERFETTO_DCHECK(cur != end); + + // Parse integer part. + int64_t int_part = 0; + for (; cur != end && std::isdigit(*cur); ++cur) { + int_part = int_part * 10 + (*cur - '0'); + } + // Parse fractional part. + double fraction = 0; + if (cur != end && *cur == '.') { + ++cur; + int64_t fract_int = 0; + uint64_t divisor = 1; + for (; cur != end && std::isdigit(*cur); ++cur) { + fract_int = (*cur - '0') + fract_int * 10; + divisor *= 10; + } + fraction = static_cast(fract_int) / static_cast(divisor); + } + // Parse exponent part. + int64_t exponent_part = 0; + bool exp_neg = false; + if (cur != end && (*cur == 'e' || *cur == 'E')) { + ++cur; + if (cur != end && (*cur == '+' || *cur == '-')) { + exp_neg = *cur++ == '-'; + } + PERFETTO_DCHECK(cur != end); + for (; cur != end && std::isdigit(*cur); ++cur) { + exponent_part = exponent_part * 10 + (*cur - '0'); + } + } + // Combine parts. + out = static_cast(int_part) + fraction; + if (exp_neg) { + out /= std::pow(10, static_cast(exponent_part)); + } else if (exponent_part > 0) { + out *= std::pow(10, static_cast(exponent_part)); + } + // Check for infinity or NaN, which indicates an overflow/underflow during pow + // or multiplication. + if (std::isinf(out) || std::isnan(out)) { + status = base::ErrStatus("Double overflow/underflow parsing '%.*s'", + int(end - start), start); + return ReturnCode::kError; + } + out = negative ? -out : out; + return ReturnCode::kOk; +} + +// Parses a JSON number, which can be an integer or a double. +// |start| and |end| define the input buffer, |cur| points to the start of the +// number. |out| is updated to point after the parsed number. |out_num| stores +// the parsed JsonValue (either int64_t or double). Sets |status| on error. +inline ReturnCode ParseNumber(const char* start, + const char* end, + const char*& out, + JsonValue& out_num, + base::Status& status) { + const char* cur = start; + PERFETTO_DCHECK(cur != end); + + bool is_int_like = true; + // Skip optional minus sign. + cur += *cur == '-'; + // Handle leading zero: only allowed if it's the only digit before '.', 'e', + // or end. + if (cur != end && *cur == '0') { + ++cur; + // "01" is invalid. + if (cur != end && std::isdigit(*cur)) { + status = base::ErrStatus("Invalid number: leading zero in '%.*s'", + int(end - start), start); + return ReturnCode::kError; + } + } else if (cur != end && *cur >= '1' && *cur <= '9') { + ++cur; + while (cur != end && std::isdigit(*cur)) { + ++cur; + } + } else if (cur != end) { + status = base::ErrStatus("Invalid number: expected digit in '%.*s'", + int(end - start), start); + return ReturnCode::kError; + } + // Check for fractional part. + if (cur != end && *cur == '.') { + is_int_like = false; + const char* frac_start_pos = ++cur; + while (cur != end && std::isdigit(*cur)) { + ++cur; + } + // Must have at least one digit after '.'. + if (cur != end && cur == frac_start_pos) { + status = + base::ErrStatus("Invalid number: expected digit after '.' in '%.*s'", + int(end - start), start); + return ReturnCode::kError; + } + } + // Check for exponent part. + if (cur != end && (*cur == 'e' || *cur == 'E')) { + is_int_like = false; + ++cur; + // Optional sign for exponent. + cur += cur != end && (*cur == '+' || *cur == '-'); + const char* exp_start_pos = cur; + while (cur != end && std::isdigit(*cur)) { + ++cur; + } + // Must have at least one digit after 'e' or 'E' (and optional sign). + if (cur != end && cur == exp_start_pos) { + status = + base::ErrStatus("Invalid number: expected digit after 'e' in '%.*s'", + int(end - start), start); + return ReturnCode::kError; + } + } + // If end is reached before any non-numeric character, input is incomplete. + if (PERFETTO_UNLIKELY(cur == end)) { + return ReturnCode::kIncompleteInput; + } + + // Attempt to parse as int64_t if it looked like an integer. + if (is_int_like) { + int64_t i_val; + // The segment [start, cur) contains the number string. + if (auto e = StringToInt64(start, cur, i_val, status); + e == ReturnCode::kOk) { + out_num = i_val; + out = cur; + return ReturnCode::kOk; + } + // If StringToInt64 failed (e.g. overflow), status is already set. + // We might still try to parse as double if it's a large integer. + // JSON spec doesn't limit integer precision, but we store as int64 or + // double. If it overflows int64, it MUST be parsed as double. + } + + // Parse as double (either because it wasn't int-like or int parsing + // failed/overflowed). + double d_val; + if (auto e = StringToDouble(start, cur, d_val, status); + e != ReturnCode::kOk) { + // If StringToInt64 failed AND StringToDouble failed, return the error from + // StringToDouble. + return e; + } + out_num = d_val; + out = cur; + return ReturnCode::kOk; +} + +// Parses the next JSON value from the input stream. +// |cur| is an in/out parameter pointing to the current position in the buffer. +// |end| points to the end of the buffer. +// |value| stores the parsed JsonValue. +// |unescaped_str| is a buffer for unescaping strings. +// Sets |status| on error. +inline ReturnCode ParseValue(const char*& cur, + const char* end, + JsonValue& value, + std::string& unescaped_str, + base::Status& status) { + const char* start = cur; + PERFETTO_CHECK(start != end); + switch (*cur) { + case '{': { + auto e = internal::ScanToEndOfDelimitedBlock(start, end, '{', '}', cur, + status); + value = Object{std::string_view(start, static_cast(cur - start))}; + return e; + } + case '[': { + auto e = internal::ScanToEndOfDelimitedBlock(start, end, '[', ']', cur, + status); + value = Array{std::string_view(start, static_cast(cur - start))}; + return e; + } + case '"': + value = std::string_view(); + return internal::ParseString(start, end, cur, + base::unchecked_get(value), + unescaped_str, status); + case 't': + if (static_cast(end - start) < 4) { + return ReturnCode::kIncompleteInput; + } + if (std::string_view(start, 4) != "true") { + status = + base::ErrStatus("Invalid token: expected 'true' but got '%.*s'", + std::min(4, static_cast(end - start)), start); + return ReturnCode::kError; + } + cur += 4; + value = true; + return ReturnCode::kOk; + case 'f': + if (static_cast(end - start) < 5) { + return ReturnCode::kIncompleteInput; + } + if (std::string_view(start, 5) != "false") { + status = + base::ErrStatus("Invalid token: expected 'false' but got '%.*s'", + std::min(5, static_cast(end - start)), start); + return ReturnCode::kError; + } + cur += 5; + value = false; + return ReturnCode::kOk; + case 'n': + if (static_cast(end - start) < 4) { + return ReturnCode::kIncompleteInput; + } + if (std::string_view(start, 4) != "null") { + status = + base::ErrStatus("Invalid token: expected 'null' but got '%.*s'", + std::min(4, static_cast(end - start)), start); + return ReturnCode::kError; + } + cur += 4; + value = Null{}; + return ReturnCode::kOk; + default: + return internal::ParseNumber(start, end, cur, value, status); + } +} + +} // namespace internal + +// Public return codes for the Iterator. +enum class ReturnCode : uint8_t { + kOk = uint8_t(internal::ReturnCode::kOk), + kError = uint8_t(internal::ReturnCode::kError), + kIncompleteInput = uint8_t(internal::ReturnCode::kIncompleteInput), + // Indicates the end of the current JSON object or array scope. + kEndOfScope = 3, +}; + +// An iterator-style parser for JSON. +// Allows for token-by-token processing of a JSON structure. +class Iterator { + public: + // Type of JSON structure currently being parsed (object or array). + enum class ParseType : uint8_t { + kObject, + kArray, + }; + + // Resets the iterator to parse a new JSON string. + // |begin| and |end| define the JSON string to be parsed. + void Reset(const char* begin, const char* end) { + cur_ = begin; + end_ = end; + parse_stack_.clear(); + status_ = base::OkStatus(); + } + + // Initializes parsing. Expects the input to start with '{' or '['. + // Returns true on success, false on failure (e.g., not starting with { or [). + bool ParseStart() { + const char* cur = cur_; + // Skip any leading whitespace. + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(cur, end_))) { + // Reached end of input while expecting '{' or '['. + status_ = base::ErrStatus( + "Expected '{' or '[' at the start. Input is empty or whitespace " + "only."); + return false; + } + // Determine if it's an object or array and push to stack. + if (*cur == '{') { + parse_stack_.push_back(ParseType::kObject); + } else if (*cur == '[') { + parse_stack_.push_back(ParseType::kArray); + } else { + status_ = + base::ErrStatus("Expected '{' or '[' at the start. Got '%c'", *cur); + return false; + } + // Skip whitespace after the opening bracket. + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(++cur, end_))) { + return false; + } + cur_ = cur; + return true; + } + + // Parses the next key-value field in an object without recursing into nested + // objects/arrays. Assumes the iterator is currently inside an object. The + // parsed key is available via `key()` and value via `value()`. Returns kOk on + // success, kEndOfScope if '}' is found, or an error code. + ReturnCode ParseObjectFieldWithoutRecursing() { + PERFETTO_DCHECK(!parse_stack_.empty()); + PERFETTO_DCHECK(parse_stack_.back() == ParseType::kObject); + const char* cur = cur_; + // Check for the end of the object. + if (PERFETTO_UNLIKELY(*cur == '}')) { + if (auto e = OnEndOfScope(cur); e != ReturnCode::kOk) { + return e; + } + cur_ = cur; + return ReturnCode::kEndOfScope; + } + // Parse the field (key: value). + if (auto e = ParseObjectFieldUntilValue(cur); e != ReturnCode::kOk) { + return e; + } + // Parse the value itself. + if (auto e = internal::ParseValue(cur, end_, value_, unescaped_str_value_, + status_); + PERFETTO_UNLIKELY(e != internal::ReturnCode::kOk)) { + return static_cast(e); + } + // Handle comma or closing brace after the value. + if (auto e = OnPostValue(cur); e != ReturnCode::kOk) { + return e; + } + cur_ = cur; + return ReturnCode::kOk; + } + + // Parses the next element. If it's an object or array, it recurses by pushing + // onto the parse stack. Otherwise, it parses the primitive value. + // The parsed key (if in an object) or value is available. + // Returns kOk on success, kEndOfScope if '}' or ']' is found, or an error + // code. + ReturnCode ParseAndRecurse() { + PERFETTO_DCHECK(!parse_stack_.empty()); + const char* cur = cur_; + // Check for end of current scope (object or array). + if (PERFETTO_UNLIKELY(*cur == '}' || *cur == ']')) { + if (auto e = OnEndOfScope(cur); e != ReturnCode::kOk) { + return e; + } + cur_ = cur; + return ReturnCode::kEndOfScope; + } + // If current scope is an object, parse the key first. + if (PERFETTO_LIKELY(parse_stack_.back() == ParseType::kObject)) { + if (auto e = ParseObjectFieldUntilValue(cur); + PERFETTO_UNLIKELY(e != ReturnCode::kOk)) { + return e; + } + } else { + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(cur, end_))) { + return ReturnCode::kIncompleteInput; + } + } + + // If the value is a new object or array, push to stack. + if (*cur == '{') { + parse_stack_.push_back(ParseType::kObject); + // Value becomes an empty Object marker; its content isn't scanned yet + // here. + value_ = Object{std::string_view()}; + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(++cur, end_))) { + return ReturnCode::kIncompleteInput; + } + cur_ = cur; + return ReturnCode::kOk; + } + if (*cur == '[') { + parse_stack_.push_back(ParseType::kArray); + // Value becomes an empty Array marker. + value_ = Array{std::string_view()}; + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(++cur, end_))) { + return ReturnCode::kIncompleteInput; + } + cur_ = cur; + return ReturnCode::kOk; + } + // Otherwise, parse the primitive value. + if (auto e = internal::ParseValue(cur, end_, value_, unescaped_str_value_, + status_); + PERFETTO_UNLIKELY(e != internal::ReturnCode::kOk)) { + return static_cast(e); + } + // Handle comma or closing brace/bracket after the value. + if (auto e = OnPostValue(cur); e != ReturnCode::kOk) { + return e; + } + cur_ = cur; + return ReturnCode::kOk; + } + + // Returns the key of the last parsed object field. + std::string_view key() const { return key_; } + // Returns the value of the last parsed field or array element. + const JsonValue& value() const { return value_; } + // Returns the current parsing position in the input buffer. + const char* cur() const { return cur_; } + // Returns the status of the last operation (Ok or an error). + const base::Status& status() const { return status_; } + + // Returns true if the entire JSON structure has been parsed (parse stack is + // empty). + bool eof() const { return parse_stack_.empty(); } + // Returns the current parse stack (e.g., for debugging or context). + const std::vector& parse_stack() const { return parse_stack_; } + + private: + // Parses an object field up to the value (i.e., "key": ). + // |cur| is advanced past the ':'. + ReturnCode ParseObjectFieldUntilValue(const char*& cur) { + // Skip whitespace before the key. + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(cur, end_))) { + return ReturnCode::kIncompleteInput; + } + // Expect a string key. + if (PERFETTO_UNLIKELY(*cur != '"')) { + status_ = + base::ErrStatus("Expected '\"' at the start of key. Got '%c'", *cur); + return ReturnCode::kError; + } + if (auto e = internal::ParseString(cur, end_, cur, key_, unescaped_key_, + status_); + PERFETTO_UNLIKELY(e != internal::ReturnCode::kOk)) { + return static_cast(e); + } + // Skip whitespace after the key. + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(cur, end_))) { + return ReturnCode::kIncompleteInput; + } + // Expect a colon separator. + if (PERFETTO_UNLIKELY(*cur != ':')) { + status_ = base::ErrStatus("Expected ':' after key '%.*s'. Got '%c'", + int(key_.size()), key_.data(), *cur); + return ReturnCode::kError; + } + // Skip whitespace after the colon. + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(++cur, end_))) { + return ReturnCode::kIncompleteInput; + } + return ReturnCode::kOk; + } + + // Handles characters after a parsed value (',' or closing '}' or ']'). + // |cur| is advanced past the delimiter and subsequent whitespace. + ReturnCode OnPostValue(const char*& cur) { + PERFETTO_DCHECK(!parse_stack_.empty()); + // Skip whitespace after the value. + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(cur, end_))) { + return ReturnCode::kIncompleteInput; + } + // Determine expected end character based on current scope. + char end_char = parse_stack_.back() == ParseType::kObject ? '}' : ']'; + // If comma, consume it and skip whitespace. + if (PERFETTO_LIKELY(*cur == ',')) { + ++cur; + if (PERFETTO_UNLIKELY(!internal::SkipWhitespace(cur, end_))) { + return ReturnCode::kIncompleteInput; + } + } else if (PERFETTO_UNLIKELY(*cur != end_char)) { + // If not a comma, it must be the end character for the current scope. + status_ = base::ErrStatus("Expected ',' or '%c' after value. Got '%c'", + end_char, *cur); + // If we are in an object, the key_ context is relevant. + if (parse_stack_.back() == ParseType::kObject && !key_.empty()) { + status_ = base::ErrStatus( + "Expected ',' or '%c' after value for key '%.*s'. Got '%c'", + end_char, int(key_.size()), key_.data(), *cur); + } + return ReturnCode::kError; + } + // If it was end_char, it will be handled by ParseAndRecurse or + // ParseObjectFieldWithoutRecursing in the next iteration, or by + // OnEndOfScope. + return ReturnCode::kOk; + } + + // Handles the end of a scope ('}' or ']'), pops from parse stack. + // |cur| is advanced past the closing delimiter. + ReturnCode OnEndOfScope(const char*& cur) { + if (PERFETTO_UNLIKELY(parse_stack_.empty())) { + status_ = base::ErrStatus("Parse stack is empty on end of scope"); + return ReturnCode::kError; + } + ++cur; // Consume '}' or ']'. + parse_stack_.pop_back(); + // If not at the end of the entire JSON (i.e., stack is not empty), + // then this scope was nested. We need to handle post-value for the parent. + if (!parse_stack_.empty()) { + if (auto e = OnPostValue(cur); e != ReturnCode::kOk) { + return e; + } + } + return ReturnCode::kOk; + } + + // Pointer to the current parsing position in the input buffer. + const char* cur_; + // Pointer to the end of the input buffer. + const char* end_; + // Holds the most recently parsed object key. + std::string_view key_; + // Buffer for unescaped key string, if key_ contains escapes. + std::string unescaped_key_; + // Buffer for unescaped value string, if value_ (as string_view) contains + // escapes. + std::string unescaped_str_value_; + // Holds the most recently parsed JSON value. + JsonValue value_; + // Stores the success/failure status of parsing operations. + base::Status status_; + // Stack to keep track of nested JSON structures (objects/arrays). + std::vector parse_stack_; +}; + +} // namespace perfetto::trace_processor::json + +#endif // SRC_TRACE_PROCESSOR_IMPORTERS_JSON_JSON_PARSER_H_ diff --git a/src/trace_processor/importers/json/json_trace_parser_impl.cc b/src/trace_processor/importers/json/json_trace_parser_impl.cc index 049d7c07514..f8478e32621 100644 --- a/src/trace_processor/importers/json/json_trace_parser_impl.cc +++ b/src/trace_processor/importers/json/json_trace_parser_impl.cc @@ -18,15 +18,19 @@ #include #include +#include #include #include -#include +#include +#include +#include "perfetto/base/compiler.h" #include "perfetto/base/logging.h" #include "perfetto/base/status.h" -#include "perfetto/ext/base/hash.h" #include "perfetto/ext/base/string_utils.h" #include "perfetto/ext/base/string_view.h" +#include "src/trace_processor/containers/null_term_string_view.h" +#include "src/trace_processor/containers/string_pool.h" #include "src/trace_processor/importers/common/event_tracker.h" #include "src/trace_processor/importers/common/flow_tracker.h" #include "src/trace_processor/importers/common/legacy_v8_cpu_profile_tracker.h" @@ -36,7 +40,7 @@ #include "src/trace_processor/importers/common/track_tracker.h" #include "src/trace_processor/importers/common/tracks.h" #include "src/trace_processor/importers/common/tracks_common.h" -#include "src/trace_processor/importers/common/tracks_internal.h" +#include "src/trace_processor/importers/json/json_parser.h" #include "src/trace_processor/importers/json/json_utils.h" #include "src/trace_processor/importers/systrace/systrace_line.h" #include "src/trace_processor/storage/stats.h" @@ -49,18 +53,29 @@ namespace perfetto::trace_processor { namespace { -std::optional MaybeExtractFlowIdentifier(const Json::Value& value, +std::optional MaybeExtractFlowIdentifier(StringPool* pool, + const JsonEvent& value, bool version2) { - std::string id_key = (version2 ? "bind_id" : "id"); - if (!value.isMember(id_key)) - return std::nullopt; - const auto& id = value[id_key]; - if (id.isNumeric()) - return id.asUInt64(); - if (!id.isString()) - return std::nullopt; - const char* c_string = id.asCString(); - return base::CStringToUInt64(c_string, 16); + const auto& id_key = version2 ? value.bind_id : value.id; + auto id_type = version2 ? value.bind_id_type : value.id_type; + switch (id_type) { + case JsonEvent::IdType::kNone: + return std::nullopt; + case JsonEvent::IdType::kString: { + NullTermStringView str = pool->Get(id_key.id_str); + return base::CStringToUInt64(str.c_str(), 16); + } + case JsonEvent::IdType::kUint64: + return id_key.id_uint64; + } + PERFETTO_FATAL("For GCC"); +} + +inline std::string_view GetStringValue(const json::JsonValue& value) { + if (const auto* str = std::get_if(&value)) { + return *str; + } + return {}; } } // namespace @@ -74,111 +89,63 @@ void JsonTraceParserImpl::ParseSystraceLine(int64_t, SystraceLine line) { systrace_line_parser_.ParseLine(line); } -void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, - std::string string_value) { - PERFETTO_DCHECK(json::IsJsonSupported()); - - auto opt_value = json::ParseJsonString(base::StringView(string_value)); - if (!opt_value) { - context_->storage->IncrementStats(stats::json_parser_failure); - return; - } - +void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, JsonEvent event) { ProcessTracker* procs = context_->process_tracker.get(); TraceStorage* storage = context_->storage.get(); SliceTracker* slice_tracker = context_->slice_tracker.get(); FlowTracker* flow_tracker = context_->flow_tracker.get(); - const Json::Value& value = *opt_value; - const auto& ph = value["ph"]; - if (!ph.isString()) - return; - char phase = *ph.asCString(); - - std::optional opt_pid; - if (value.isMember("pid")) { - if (value["pid"].isString()) { - // If the pid is a string, treat raw id of the interned string as the pid. - // This "hack" which allows emitting "quick-and-dirty" compact JSON - // traces: relying on these traces for production is necessarily brittle - // as it is not a part of the actual spec. - const char* proc_name = value["pid"].asCString(); - opt_pid = storage->InternString(proc_name).raw_id(); - procs->SetProcessMetadata(*opt_pid, std::nullopt, proc_name, - base::StringView()); - } else { - opt_pid = json::CoerceToUint32(value["pid"]); - } + if (event.pid_is_string_id) { + procs->SetProcessMetadata( + event.pid, std::nullopt, + storage->GetString(StringPool::Id::Raw(event.pid)), base::StringView()); } - - std::optional opt_tid; - if (value.isMember("tid")) { - if (value["tid"].isString()) { - // See the comment for |pid| string handling above: the same applies here. - StringId thread_name_id = storage->InternString(value["tid"].asCString()); - opt_tid = thread_name_id.raw_id(); - procs->UpdateThreadName(*opt_tid, thread_name_id, - ThreadNamePriority::kOther); - } else { - opt_tid = json::CoerceToUint32(value["tid"]); - } + if (event.tid_is_string_id) { + procs->UpdateThreadName(event.tid, StringPool::Id::Raw(event.tid), + ThreadNamePriority::kOther); } - - uint32_t pid = opt_pid.value_or(0); - uint32_t tid = opt_tid.value_or(pid); - UniqueTid utid = procs->UpdateThread(tid, pid); - - std::string id = value.isMember("id") ? value["id"].asString() : ""; - - base::StringView cat = value.isMember("cat") && value["cat"].isString() - ? base::StringView(value["cat"].asCString()) - : base::StringView(); - StringId cat_id = storage->InternString(cat); - - base::StringView name = value.isMember("name") - ? base::StringView(value["name"].asCString()) - : base::StringView(); - StringId name_id = name.empty() ? kNullStringId : storage->InternString(name); - - auto args_inserter = [this, &value](ArgsTracker::BoundInserter* inserter) { - if (value.isMember("args")) { - json::AddJsonValueToArgs(value["args"], /* flat_key = */ "args", - /* key = */ "args", context_->storage.get(), - inserter); - } - }; + UniqueTid utid = procs->UpdateThread(event.tid, event.pid); // Only used for 'B', 'E', and 'X' events so wrap in lambda so it gets // ignored in other cases. This lambda is only safe to call within the // scope of this function due to the capture by reference. - StringId slice_name_id = - name_id == kNullStringId ? storage->InternString("[No name]") : name_id; - switch (phase) { + auto args_inserter = [&](ArgsTracker::BoundInserter* inserter) { + if (event.args_size > 0) { + json::AddJsonValueToArgs( + it_, event.args.get(), event.args.get() + event.args_size, + /* flat_key = */ "args", + /* key = */ "args", context_->storage.get(), inserter); + } + }; + + base::StringView id; + if (event.id_type == JsonEvent::IdType::kString) { + id = context_->storage->GetString(event.id.id_str); + } + StringId slice_name_id = event.name == kNullStringId + ? storage->InternString("[No name]") + : event.name; + switch (event.phase) { case 'B': { // TRACE_EVENT_BEGIN. TrackId track_id = context_->track_tracker->InternThreadTrack(utid); - auto slice_id = slice_tracker->Begin(timestamp, track_id, cat_id, + auto slice_id = slice_tracker->Begin(timestamp, track_id, event.cat, slice_name_id, args_inserter); - if (slice_id) { - if (auto thread_ts = json::CoerceToTs(value["tts"]); thread_ts) { - auto rr = - context_->storage->mutable_slice_table()->FindById(*slice_id); - rr->set_thread_ts(*thread_ts); - } + if (slice_id && event.tts != std::numeric_limits::max()) { + auto rr = context_->storage->mutable_slice_table()->FindById(*slice_id); + rr->set_thread_ts(event.tts); } - MaybeAddFlow(track_id, value); + MaybeAddFlow(storage->mutable_string_pool(), track_id, event); break; } case 'E': { // TRACE_EVENT_END. TrackId track_id = context_->track_tracker->InternThreadTrack(utid); - auto opt_slice_id = slice_tracker->End(timestamp, track_id, cat_id, - name_id, args_inserter); + auto slice_id = slice_tracker->End(timestamp, track_id, event.cat, + event.name, args_inserter); // Now try to update thread_dur if we have a tts field. - auto opt_tts = json::CoerceToTs(value["tts"]); - if (opt_slice_id.has_value() && opt_tts) { - auto* slice = storage->mutable_slice_table(); - auto rr = *slice->FindById(*opt_slice_id); + if (slice_id && event.tts != std::numeric_limits::max()) { + auto rr = *storage->mutable_slice_table()->FindById(*slice_id); if (auto start_tts = rr.thread_ts(); start_tts) { - rr.set_thread_dur(*opt_tts - *start_tts); + rr.set_thread_dur(event.tts - *start_tts); } } break; @@ -186,94 +153,119 @@ void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, case 'b': case 'e': case 'n': { - Json::Value id2 = value.isMember("id2") ? value["id2"] : Json::Value(); - std::string local = id2.isMember("local") ? id2["local"].asString() : ""; - std::string global = - id2.isMember("global") ? id2["global"].asString() : ""; - if (!opt_pid || (id.empty() && global.empty() && local.empty())) { + if (event.pid == 0 || + event.async_cookie == std::numeric_limits::max()) { context_->storage->IncrementStats(stats::json_parser_failure); - break; + return; } - UniquePid upid = context_->process_tracker->GetOrCreateProcess(pid); + UniquePid upid = context_->process_tracker->GetOrCreateProcess(event.pid); TrackId track_id; - if (!id.empty() || !global.empty()) { - const std::string& real_id = id.empty() ? global : id; - int64_t cookie = static_cast( - base::Hasher::Combine(cat_id.raw_id(), real_id)); + if (event.async_cookie_type == JsonEvent::AsyncCookieType::kId || + event.async_cookie_type == JsonEvent::AsyncCookieType::kId2Global) { track_id = context_->track_tracker->InternLegacyAsyncTrack( - name_id, upid, cookie, false /* source_id_is_process_scoped */, + event.name, upid, event.async_cookie, + false /* source_id_is_process_scoped */, kNullStringId /* source_scope */); } else { - PERFETTO_DCHECK(!local.empty()); - int64_t cookie = - static_cast(base::Hasher::Combine(cat_id.raw_id(), local)); + PERFETTO_DCHECK(event.async_cookie_type == + JsonEvent::AsyncCookieType::kId2Local); track_id = context_->track_tracker->InternLegacyAsyncTrack( - name_id, upid, cookie, true /* source_id_is_process_scoped */, + event.name, upid, event.async_cookie, + true /* source_id_is_process_scoped */, kNullStringId /* source_scope */); } - - if (phase == 'b') { - slice_tracker->Begin(timestamp, track_id, cat_id, slice_name_id, + if (event.phase == 'b') { + slice_tracker->Begin(timestamp, track_id, event.cat, slice_name_id, args_inserter); - MaybeAddFlow(track_id, value); - } else if (phase == 'e') { - slice_tracker->End(timestamp, track_id, cat_id, name_id, args_inserter); + MaybeAddFlow(storage->mutable_string_pool(), track_id, event); + } else if (event.phase == 'e') { + slice_tracker->End(timestamp, track_id, event.cat, event.name, + args_inserter); // We don't handle tts here as we do in the 'E' // case above as it's not well defined for async slices. } else { - context_->slice_tracker->Scoped(timestamp, track_id, cat_id, name_id, 0, - args_inserter); - MaybeAddFlow(track_id, value); + context_->slice_tracker->Scoped(timestamp, track_id, event.cat, + event.name, 0, args_inserter); + MaybeAddFlow(storage->mutable_string_pool(), track_id, event); } break; } case 'X': { // TRACE_EVENT (scoped event). - std::optional opt_dur = json::CoerceToTs(value["dur"]); - if (!opt_dur.has_value()) + if (event.dur == std::numeric_limits::max()) { + context_->storage->IncrementStats(stats::json_parser_failure); return; + } TrackId track_id = context_->track_tracker->InternThreadTrack(utid); - auto slice_id = slice_tracker->Scoped( - timestamp, track_id, cat_id, slice_name_id, *opt_dur, args_inserter); + auto slice_id = + slice_tracker->Scoped(timestamp, track_id, event.cat, slice_name_id, + event.dur, args_inserter); if (slice_id) { auto rr = context_->storage->mutable_slice_table()->FindById(*slice_id); - if (auto thread_ts = json::CoerceToTs(value["tts"]); thread_ts) { - rr->set_thread_ts(*thread_ts); + if (event.tts != std::numeric_limits::max()) { + rr->set_thread_ts(event.tts); } - if (auto thread_dur = json::CoerceToTs(value["tdur"]); thread_dur) { - rr->set_thread_dur(*thread_dur); + if (event.tdur != std::numeric_limits::max()) { + rr->set_thread_dur(event.tdur); } } - MaybeAddFlow(track_id, value); + MaybeAddFlow(storage->mutable_string_pool(), track_id, event); break; } case 'C': { // TRACE_EVENT_COUNTER - auto args = value["args"]; - if (!args.isObject()) { + if (event.args_size == 0) { context_->storage->IncrementStats(stats::json_parser_failure); - break; + return; } - - std::string counter_name_prefix = name.ToStdString(); + it_.Reset(event.args.get(), event.args.get() + event.args_size); + if (!it_.ParseStart()) { + context_->storage->IncrementStats(stats::json_parser_failure); + return; + } + std::string counter_name_prefix = + context_->storage->GetString(event.name).ToStdString(); if (!id.empty()) { - counter_name_prefix += " id: " + id; + counter_name_prefix += " id: " + id.ToStdString(); } - - for (auto it = args.begin(); it != args.end(); ++it) { + counter_name_prefix += " "; + for (;;) { double counter; - if (it->isString()) { - auto opt = base::CStringToDouble(it->asCString()); - if (!opt.has_value()) { + switch (it_.ParseObjectFieldWithoutRecursing()) { + case json::ReturnCode::kOk: + case json::ReturnCode::kEndOfScope: + break; + case json::ReturnCode::kError: context_->storage->IncrementStats(stats::json_parser_failure); continue; + case json::ReturnCode::kIncompleteInput: + PERFETTO_FATAL("Unexpected incomplete input in JSON object"); + } + if (it_.eof()) { + break; + } + switch (it_.value().index()) { + case base::variant_index(): { + auto opt = base::StringToDouble(std::string( + base::unchecked_get(it_.value()))); + if (!opt.has_value()) { + context_->storage->IncrementStats(stats::json_parser_failure); + continue; + } + counter = opt.value(); + break; } - counter = opt.value(); - } else if (it->isNumeric()) { - counter = it->asDouble(); - } else { - context_->storage->IncrementStats(stats::json_parser_failure); - continue; + case base::variant_index(): + counter = base::unchecked_get(it_.value()); + break; + case base::variant_index(): + counter = + static_cast(base::unchecked_get(it_.value())); + break; + default: + context_->storage->IncrementStats(stats::json_parser_failure); + continue; } - std::string counter_name = counter_name_prefix + " " + it.name(); + std::string counter_name = counter_name_prefix; + counter_name += it_.key(); StringId nid = context_->storage->InternString(counter_name); context_->event_tracker->PushProcessCounterForThread( EventTracker::JsonCounter{nid}, timestamp, counter, utid); @@ -283,13 +275,8 @@ void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, case 'R': case 'I': case 'i': { // TRACE_EVENT_INSTANT - base::StringView scope; - if (value.isMember("s")) { - scope = value["s"].asCString(); - } - TrackId track_id; - if (scope == "g") { + if (event.scope == JsonEvent::Scope::kGlobal) { track_id = context_->track_tracker->InternTrack( tracks::kLegacyGlobalInstantsBlueprint, tracks::Dimensions(), tracks::BlueprintName(), @@ -298,12 +285,13 @@ void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, context_->storage->InternString("source"), Variadic::String(context_->storage->InternString("chrome"))); }); - } else if (scope == "p") { - if (!opt_pid) { + } else if (event.scope == JsonEvent::Scope::kProcess) { + if (event.pid == 0) { context_->storage->IncrementStats(stats::json_parser_failure); break; } - UniquePid upid = context_->process_tracker->GetOrCreateProcess(pid); + UniquePid upid = + context_->process_tracker->GetOrCreateProcess(event.pid); track_id = context_->track_tracker->InternTrack( tracks::kChromeProcessInstantBlueprint, tracks::Dimensions(upid), tracks::BlueprintName(), @@ -312,37 +300,39 @@ void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, context_->storage->InternString("source"), Variadic::String(context_->storage->InternString("chrome"))); }); - } else if (scope == "t" || scope.data() == nullptr) { - if (!opt_tid) { + } else if (event.scope == JsonEvent::Scope::kThread || + event.scope == JsonEvent::Scope::kNone) { + if (event.tid == 0) { context_->storage->IncrementStats(stats::json_parser_failure); - break; + return; } track_id = context_->track_tracker->InternThreadTrack(utid); - auto slice_id = slice_tracker->Scoped(timestamp, track_id, cat_id, + auto slice_id = slice_tracker->Scoped(timestamp, track_id, event.cat, slice_name_id, 0, args_inserter); if (slice_id) { - if (auto thread_ts = json::CoerceToTs(value["tts"]); thread_ts) { + if (event.tts != std::numeric_limits::max()) { auto rr = context_->storage->mutable_slice_table()->FindById(*slice_id); - rr->set_thread_ts(*thread_ts); + rr->set_thread_ts(event.tts); } } break; } else { context_->storage->IncrementStats(stats::json_parser_failure); - break; + return; } - context_->slice_tracker->Scoped(timestamp, track_id, cat_id, name_id, 0, - args_inserter); + context_->slice_tracker->Scoped(timestamp, track_id, event.cat, + event.name, 0, args_inserter); break; } case 's': { // TRACE_EVENT_FLOW_START TrackId track_id = context_->track_tracker->InternThreadTrack(utid); auto opt_source_id = - MaybeExtractFlowIdentifier(value, /* version2 = */ false); + MaybeExtractFlowIdentifier(context_->storage->mutable_string_pool(), + event, /* version2 = */ false); if (opt_source_id) { FlowId flow_id = flow_tracker->GetFlowIdForV1Event( - opt_source_id.value(), cat_id, name_id); + opt_source_id.value(), event.cat, event.name); flow_tracker->Begin(track_id, flow_id); } else { context_->storage->IncrementStats(stats::flow_invalid_id); @@ -352,10 +342,11 @@ void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, case 't': { // TRACE_EVENT_FLOW_STEP TrackId track_id = context_->track_tracker->InternThreadTrack(utid); auto opt_source_id = - MaybeExtractFlowIdentifier(value, /* version2 = */ false); + MaybeExtractFlowIdentifier(context_->storage->mutable_string_pool(), + event, /* version2 = */ false); if (opt_source_id) { FlowId flow_id = flow_tracker->GetFlowIdForV1Event( - opt_source_id.value(), cat_id, name_id); + opt_source_id.value(), event.cat, event.name); flow_tracker->Step(track_id, flow_id); } else { context_->storage->IncrementStats(stats::flow_invalid_id); @@ -365,13 +356,12 @@ void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, case 'f': { // TRACE_EVENT_FLOW_END TrackId track_id = context_->track_tracker->InternThreadTrack(utid); auto opt_source_id = - MaybeExtractFlowIdentifier(value, /* version2 = */ false); + MaybeExtractFlowIdentifier(context_->storage->mutable_string_pool(), + event, /* version2 = */ false); if (opt_source_id) { FlowId flow_id = flow_tracker->GetFlowIdForV1Event( - opt_source_id.value(), cat_id, name_id); - bool bind_enclosing_slice = - value.isMember("bp") && strcmp(value["bp"].asCString(), "e") == 0; - flow_tracker->End(track_id, flow_id, bind_enclosing_slice, + opt_source_id.value(), event.cat, event.name); + flow_tracker->End(track_id, flow_id, event.bind_enclosing_slice, /* close_flow = */ false); } else { context_->storage->IncrementStats(stats::flow_invalid_id); @@ -379,36 +369,67 @@ void JsonTraceParserImpl::ParseJsonPacket(int64_t timestamp, break; } case 'M': { // Metadata events (process and thread names). - if (name == "thread_name" && !value["args"]["name"].empty()) { - const char* thread_name = value["args"]["name"].asCString(); - auto thread_name_id = context_->storage->InternString(thread_name); - procs->UpdateThreadName(tid, thread_name_id, - ThreadNamePriority::kOther); + if (event.args_size == 0) { + break; + } + base::StringView name = storage->GetString(event.name); + if (name != "thread_name" && name != "process_name") { break; } - if (name == "process_name" && !value["args"]["name"].empty()) { - const char* proc_name = value["args"]["name"].asCString(); - procs->SetProcessMetadata(pid, std::nullopt, proc_name, - base::StringView()); + it_.Reset(event.args.get(), event.args.get() + event.args_size); + if (!it_.ParseStart()) { + context_->storage->IncrementStats(stats::json_parser_failure); break; } + for (;;) { + switch (it_.ParseObjectFieldWithoutRecursing()) { + case json::ReturnCode::kEndOfScope: + case json::ReturnCode::kOk: + break; + case json::ReturnCode::kError: + context_->storage->IncrementStats(stats::json_parser_failure); + continue; + case json::ReturnCode::kIncompleteInput: + PERFETTO_FATAL("Unexpected incomplete input in JSON object"); + } + if (it_.eof()) { + break; + } + if (it_.key() != "name") { + continue; + } + std::string_view args_name = GetStringValue(it_.value()); + if (args_name.empty()) { + context_->storage->IncrementStats(stats::json_parser_failure); + continue; + } + if (name == "thread_name") { + auto thread_name_id = context_->storage->InternString(args_name); + procs->UpdateThreadName(event.tid, thread_name_id, + ThreadNamePriority::kOther); + } else if (name == "process_name") { + procs->SetProcessMetadata( + event.pid, std::nullopt, + base::StringView(args_name.data(), args_name.size()), + base::StringView()); + } + } } } } -void JsonTraceParserImpl::MaybeAddFlow(TrackId track_id, - const Json::Value& event) { - PERFETTO_DCHECK(json::IsJsonSupported()); - auto opt_bind_id = MaybeExtractFlowIdentifier(event, /* version2 = */ true); +void JsonTraceParserImpl::MaybeAddFlow(StringPool* pool, + TrackId track_id, + const JsonEvent& event) { + auto opt_bind_id = + MaybeExtractFlowIdentifier(pool, event, /* version2 = */ true); if (opt_bind_id) { FlowTracker* flow_tracker = context_->flow_tracker.get(); - bool flow_out = event.isMember("flow_out") && event["flow_out"].asBool(); - bool flow_in = event.isMember("flow_in") && event["flow_in"].asBool(); - if (flow_in && flow_out) { + if (event.flow_in && event.flow_out) { flow_tracker->Step(track_id, opt_bind_id.value()); - } else if (flow_out) { + } else if (event.flow_out) { flow_tracker->Begin(track_id, opt_bind_id.value()); - } else if (flow_in) { + } else if (event.flow_in) { // bind_enclosing_slice is always true for v2 flow events flow_tracker->End(track_id, opt_bind_id.value(), true, /* close_flow = */ false); diff --git a/src/trace_processor/importers/json/json_trace_parser_impl.h b/src/trace_processor/importers/json/json_trace_parser_impl.h index 8b89897b75b..eb351676e2e 100644 --- a/src/trace_processor/importers/json/json_trace_parser_impl.h +++ b/src/trace_processor/importers/json/json_trace_parser_impl.h @@ -18,17 +18,15 @@ #define SRC_TRACE_PROCESSOR_IMPORTERS_JSON_JSON_TRACE_PARSER_IMPL_H_ #include -#include +#include "src/trace_processor/containers/string_pool.h" +#include "src/trace_processor/importers/common/parser_types.h" #include "src/trace_processor/importers/common/trace_parser.h" +#include "src/trace_processor/importers/json/json_parser.h" #include "src/trace_processor/importers/systrace/systrace_line.h" #include "src/trace_processor/importers/systrace/systrace_line_parser.h" #include "src/trace_processor/storage/trace_storage.h" -namespace Json { -class Value; -} - namespace perfetto::trace_processor { class TraceProcessorContext; @@ -41,15 +39,16 @@ class JsonTraceParserImpl : public JsonTraceParser { ~JsonTraceParserImpl() override; // TraceParser implementation. - void ParseJsonPacket(int64_t, std::string) override; + void ParseJsonPacket(int64_t, JsonEvent) override; void ParseSystraceLine(int64_t, SystraceLine) override; void ParseLegacyV8ProfileEvent(int64_t, LegacyV8CpuProfileEvent) override; private: TraceProcessorContext* const context_; SystraceLineParser systrace_line_parser_; + json::Iterator it_; - void MaybeAddFlow(TrackId track_id, const Json::Value& event); + void MaybeAddFlow(StringPool* pool, TrackId track_id, const JsonEvent& event); }; } // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/json/json_trace_tokenizer.cc b/src/trace_processor/importers/json/json_trace_tokenizer.cc index 83e2b3c7624..0d658b47319 100644 --- a/src/trace_processor/importers/json/json_trace_tokenizer.cc +++ b/src/trace_processor/importers/json/json_trace_tokenizer.cc @@ -17,33 +17,41 @@ #include "src/trace_processor/importers/json/json_trace_tokenizer.h" #include +#include #include #include +#include +#include #include #include #include +#include #include +#include #include +#include "perfetto/base/compiler.h" #include "perfetto/base/logging.h" #include "perfetto/base/status.h" +#include "perfetto/ext/base/hash.h" #include "perfetto/ext/base/string_utils.h" #include "perfetto/ext/base/string_view.h" #include "perfetto/public/compiler.h" #include "perfetto/trace_processor/trace_blob_view.h" #include "src/trace_processor/importers/common/legacy_v8_cpu_profile_tracker.h" #include "src/trace_processor/importers/common/parser_types.h" +#include "src/trace_processor/importers/json/json_parser.h" #include "src/trace_processor/importers/json/json_utils.h" +#include "src/trace_processor/importers/systrace/systrace_line.h" #include "src/trace_processor/sorter/trace_sorter.h" // IWYU pragma: keep #include "src/trace_processor/storage/stats.h" +#include "src/trace_processor/storage/trace_storage.h" #include "src/trace_processor/util/status_macros.h" namespace perfetto::trace_processor { namespace { -std::string FormatErrorContext(const char* s, const char* e) { - return {s, static_cast(e - s)}; -} +using State = json::ReturnCode; base::Status AppendUnescapedCharacter(char c, bool is_escaping, @@ -188,66 +196,47 @@ base::Status SetOutAndReturn(const char* ptr, const char** out) { return base::OkStatus(); } -} // namespace +uint32_t CoerceToUint32(int64_t n) { + if (n < 0 || n > std::numeric_limits::max()) { + return 0; + } + return static_cast(n); +} -ReadDictRes ReadOneJsonDict(const char* start, - const char* end, - base::StringView* value, - const char** next) { - int braces = 0; - int square_brackets = 0; - const char* dict_begin = nullptr; - bool in_string = false; - bool is_escaping = false; - for (const char* s = start; s < end; s++) { - if (isspace(*s) || *s == ',') - continue; - if (*s == '"' && !is_escaping) { - in_string = !in_string; - continue; - } - if (in_string) { - // If we're in a string and we see a backslash and the last character was - // not a backslash the next character is escaped: - is_escaping = *s == '\\' && !is_escaping; - // If we're currently parsing a string we should ignore otherwise special - // characters: - continue; - } - if (*s == '{') { - if (braces == 0) - dict_begin = s; - braces++; - continue; - } - if (*s == '}') { - if (braces <= 0) - return ReadDictRes::kEndOfTrace; - if (--braces > 0) - continue; - auto len = static_cast((s + 1) - dict_begin); - *value = base::StringView(dict_begin, len); - *next = s + 1; - return ReadDictRes::kFoundDict; - } - if (*s == '[') { - square_brackets++; - continue; - } - if (*s == ']') { - if (square_brackets == 0) { - // We've reached the end of [traceEvents] array. - // There might be other top level keys in the json (e.g. metadata) - // after. - *next = s + 1; - return ReadDictRes::kEndOfArray; - } - square_brackets--; - } +uint32_t CoerceToUint32(double n) { + return CoerceToUint32(static_cast(n)); +} + +inline int64_t CoerceToTs(const json::JsonValue& value) { + switch (value.index()) { + case base::variant_index(): + return static_cast( + std::llround(base::unchecked_get(value) * 1000.0)); + case base::variant_index(): + return base::unchecked_get(value) * 1000; + case base::variant_index(): + return CoerceToTs(base::unchecked_get(value)); + default: + return std::numeric_limits::max(); } - return ReadDictRes::kNeedsMoreData; } +inline std::string_view GetStringValue(const json::JsonValue& value) { + if (const auto* str = std::get_if(&value)) { + return *str; + } + return {}; +} + +inline std::string_view GetObjectValue(const json::JsonValue& value) { + if (const auto* o = std::get_if(&value)) { + return o->contents; + } + return {}; +} + +} // namespace + ReadKeyRes ReadOneJsonKey(const char* start, const char* end, std::string* key, @@ -273,7 +262,7 @@ ReadKeyRes ReadOneJsonKey(const char* start, return ReadKeyRes::kEndOfDictionary; } - // If we see a comma separator, just ignore it. + // If we see a comma separator, just ignore it_. if (*s == ',') continue; @@ -309,122 +298,6 @@ ReadKeyRes ReadOneJsonKey(const char* start, return ReadKeyRes::kNeedsMoreData; } -base::Status ExtractValueForJsonKey(base::StringView dict, - const std::string& key, - std::optional* value) { - PERFETTO_DCHECK(dict.size() >= 2); - - const char* start = dict.data(); - const char* end = dict.data() + dict.size(); - - enum ExtractValueState { - kBeforeDict, - kInsideDict, - kAfterDict, - }; - - ExtractValueState state = kBeforeDict; - for (const char* s = start; s < end;) { - if (isspace(*s)) { - ++s; - continue; - } - - if (state == kBeforeDict) { - if (*s == '{') { - ++s; - state = kInsideDict; - continue; - } - return base::ErrStatus("Unexpected character before JSON dict: '%c'", *s); - } - - if (state == kAfterDict) { - return base::ErrStatus("Unexpected character after JSON dict: '%c'", *s); - } - - PERFETTO_DCHECK(state == kInsideDict); - PERFETTO_DCHECK(s < end); - - if (*s == '}') { - ++s; - state = kAfterDict; - continue; - } - - std::string current_key; - auto res = ReadOneJsonKey(s, end, ¤t_key, &s); - if (res == ReadKeyRes::kEndOfDictionary) - break; - - if (res == ReadKeyRes::kFatalError) { - return base::ErrStatus( - "Failure parsing JSON: encountered fatal error while parsing key for " - "value: '%s'", - FormatErrorContext(s, end).c_str()); - } - - if (res == ReadKeyRes::kNeedsMoreData) { - return base::ErrStatus( - "Failure parsing JSON: partial JSON dictionary: '%s'", - FormatErrorContext(s, end).c_str()); - } - - PERFETTO_DCHECK(res == ReadKeyRes::kFoundKey); - - if (*s == '[') { - return base::ErrStatus( - "Failure parsing JSON: unsupported JSON dictionary with array: '%s'", - FormatErrorContext(s, end).c_str()); - } - - std::string value_str; - if (*s == '{') { - base::StringView dict_str; - ReadDictRes dict_res = ReadOneJsonDict(s, end, &dict_str, &s); - if (dict_res == ReadDictRes::kNeedsMoreData || - dict_res == ReadDictRes::kEndOfArray || - dict_res == ReadDictRes::kEndOfTrace) { - return base::ErrStatus( - "Failure parsing JSON: unable to parse dictionary: '%s'", - FormatErrorContext(s, end).c_str()); - } - value_str = dict_str.ToStdString(); - } else if (*s == '"') { - auto str_res = ReadOneJsonString(s, end, &value_str, &s); - if (str_res == ReadStringRes::kNeedsMoreData || - str_res == ReadStringRes::kFatalError) { - return base::ErrStatus( - "Failure parsing JSON: unable to parse string: '%s", - FormatErrorContext(s, end).c_str()); - } - } else { - const char* value_start = s; - const char* value_end = end; - for (; s < end; ++s) { - if (*s == ',' || isspace(*s) || *s == '}') { - value_end = s; - break; - } - } - value_str = std::string(value_start, value_end); - } - - if (key == current_key) { - *value = value_str; - return base::OkStatus(); - } - } - - if (state != kAfterDict) { - return base::ErrStatus("Failure parsing JSON: malformed dictionary: '%s'", - FormatErrorContext(start, end).c_str()); - } - - *value = std::nullopt; - return base::OkStatus(); -} - ReadSystemLineRes ReadOneSystemTraceLine(const char* start, const char* end, std::string* line, @@ -461,8 +334,6 @@ JsonTraceTokenizer::JsonTraceTokenizer(TraceProcessorContext* ctx) JsonTraceTokenizer::~JsonTraceTokenizer() = default; base::Status JsonTraceTokenizer::Parse(TraceBlobView blob) { - PERFETTO_DCHECK(json::IsJsonSupported()); - buffer_.insert(buffer_.end(), blob.data(), blob.data() + blob.size()); const char* buf = buffer_.data(); const char* next = buf; @@ -510,8 +381,6 @@ base::Status JsonTraceTokenizer::Parse(TraceBlobView blob) { base::Status JsonTraceTokenizer::ParseInternal(const char* start, const char* end, const char** out) { - PERFETTO_DCHECK(json::IsJsonSupported()); - switch (position_) { case TracePosition::kDictionaryKey: return HandleDictionaryKey(start, end, out); @@ -532,94 +401,286 @@ base::Status JsonTraceTokenizer::ParseInternal(const char* start, base::Status JsonTraceTokenizer::HandleTraceEvent(const char* start, const char* end, const char** out) { - const char* next = start; - while (next < end) { - base::StringView unparsed; - switch (ReadOneJsonDict(next, end, &unparsed, &next)) { - case ReadDictRes::kEndOfArray: { - if (format_ == TraceFormat::kOnlyTraceEvents) { - position_ = TracePosition::kEof; - return SetOutAndReturn(next, out); - } - - position_ = TracePosition::kDictionaryKey; - return ParseInternal(next, end, out); - } - case ReadDictRes::kEndOfTrace: + const char* global_cur = start; + for (;;) { + const char* cur = global_cur; + if (PERFETTO_UNLIKELY(!json::internal::SkipWhitespace(cur, end))) { + return SetOutAndReturn(global_cur, out); + } + if (PERFETTO_UNLIKELY(*cur == ']')) { + if (format_ == TraceFormat::kOnlyTraceEvents) { position_ = TracePosition::kEof; - return SetOutAndReturn(next, out); - case ReadDictRes::kNeedsMoreData: - return SetOutAndReturn(next, out); - case ReadDictRes::kFoundDict: - break; + return SetOutAndReturn(cur + 1, out); + } + position_ = TracePosition::kDictionaryKey; + return ParseInternal(cur + 1, end, out); } - - // Metadata events may omit ts. In all other cases error: - std::optional opt_raw_ph; - RETURN_IF_ERROR(ExtractValueForJsonKey(unparsed, "ph", &opt_raw_ph)); - if (PERFETTO_UNLIKELY(opt_raw_ph == "P")) { - RETURN_IF_ERROR(ParseV8SampleEvent(unparsed)); - continue; + if (PERFETTO_UNLIKELY(*cur == ',')) { + if (PERFETTO_UNLIKELY(!json::internal::SkipWhitespace(++cur, end))) { + return SetOutAndReturn(global_cur, out); + } } + it_.Reset(cur, end); + if (!it_.ParseStart() || !ParseTraceEventContents()) { + RETURN_IF_ERROR(it_.status()); + return SetOutAndReturn(global_cur, out); + } + global_cur = it_.cur(); + } +} - std::optional opt_raw_ts; - RETURN_IF_ERROR(ExtractValueForJsonKey(unparsed, "ts", &opt_raw_ts)); - std::optional opt_ts = - opt_raw_ts ? json::CoerceToTs(*opt_raw_ts) : std::nullopt; - std::optional opt_raw_dur; - RETURN_IF_ERROR(ExtractValueForJsonKey(unparsed, "dur", &opt_raw_dur)); - std::optional opt_dur = - opt_raw_dur ? json::CoerceToTs(*opt_raw_dur) : std::nullopt; - int64_t ts = 0; - if (opt_ts.has_value()) { - ts = opt_ts.value(); - } else { - if (!opt_raw_ph || *opt_raw_ph != "M") { - context_->storage->IncrementStats(stats::json_tokenizer_failure); - continue; - } +bool JsonTraceTokenizer::ParseTraceEventContents() { + JsonEvent event; + int64_t ts = std::numeric_limits::max(); + std::string_view id2_local; + std::string_view id2_global; + for (;;) { + switch (it_.ParseObjectFieldWithoutRecursing()) { + case State::kOk: + case State::kEndOfScope: + break; + case State::kIncompleteInput: + case State::kError: + return false; } - JsonEvent::Type type; - if (opt_raw_ph && opt_raw_ph->size() == 1) { - switch ((*opt_raw_ph)[0]) { - case 'B': - type = JsonEvent::Begin(); + if (it_.eof()) { + break; + } + if (it_.key() == "ph") { + std::string_view ph = GetStringValue(it_.value()); + event.phase = ph.size() >= 1 ? ph[0] : '\0'; + } else if (it_.key() == "ts") { + ts = CoerceToTs(it_.value()); + } else if (it_.key() == "dur") { + event.dur = CoerceToTs(it_.value()); + } else if (it_.key() == "pid") { + switch (it_.value().index()) { + case base::variant_index(): { + // If the pid is a string, treat raw id of the interned string as + // the pid. This "hack" which allows emitting "quick-and-dirty" + // compact JSON traces: relying on these traces for production is + // necessarily brittle as it is not a part of the actual spec. + std::string_view proc_name = + base::unchecked_get(it_.value()); + event.pid = context_->storage->InternString(proc_name).raw_id(); + event.pid_is_string_id = true; + break; + } + case base::variant_index(): + event.pid = CoerceToUint32(base::unchecked_get(it_.value())); + break; + case base::variant_index(): + event.pid = CoerceToUint32(base::unchecked_get(it_.value())); + break; + default: + break; + } + } else if (it_.key() == "tid") { + switch (it_.value().index()) { + case base::variant_index(): { + // See the comment for |pid| string handling above: the same applies + // here. + std::string_view thread_name = + base::unchecked_get(it_.value()); + event.tid = context_->storage->InternString(thread_name).raw_id(); + event.tid_is_string_id = true; + break; + } + case base::variant_index(): + event.tid = CoerceToUint32(base::unchecked_get(it_.value())); break; - case 'E': - type = JsonEvent::End(); + case base::variant_index(): + event.tid = CoerceToUint32(base::unchecked_get(it_.value())); break; - case 'X': - if (opt_dur) { - type = JsonEvent::Scoped{*opt_dur}; - } else { - type = JsonEvent::Other(); - } + default: break; } - } else { - type = JsonEvent::Other(); + } else if (it_.key() == "id") { + switch (it_.value().index()) { + case base::variant_index(): + event.id.id_str = context_->storage->InternString( + base::unchecked_get(it_.value())); + event.id_type = JsonEvent::IdType::kString; + break; + case base::variant_index(): + event.id.id_uint64 = + static_cast(base::unchecked_get(it_.value())); + event.id_type = JsonEvent::IdType::kUint64; + break; + default: + break; + } + } else if (it_.key() == "bind_id") { + switch (it_.value().index()) { + case base::variant_index(): + event.bind_id.id_str = context_->storage->InternString( + base::unchecked_get(it_.value())); + event.bind_id_type = JsonEvent::IdType::kString; + break; + case base::variant_index(): + event.bind_id.id_uint64 = + static_cast(base::unchecked_get(it_.value())); + event.bind_id_type = JsonEvent::IdType::kUint64; + break; + default: + break; + } + } else if (it_.key() == "cat") { + std::string_view cat = GetStringValue(it_.value()); + event.cat = + cat.empty() ? kNullStringId : context_->storage->InternString(cat); + } else if (it_.key() == "name") { + std::string_view name = GetStringValue(it_.value()); + event.name = + name.empty() ? kNullStringId : context_->storage->InternString(name); + } else if (it_.key() == "flow_in") { + switch (it_.value().index()) { + case base::variant_index(): + event.flow_in = base::unchecked_get(it_.value()); + break; + default: + break; + } + } else if (it_.key() == "flow_out") { + switch (it_.value().index()) { + case base::variant_index(): + event.flow_out = base::unchecked_get(it_.value()); + break; + default: + break; + } + } else if (it_.key() == "s") { + auto value = GetStringValue(it_.value()); + if (value == "p") { + event.scope = JsonEvent::Scope::kProcess; + } else if (value == "t") { + event.scope = JsonEvent::Scope::kThread; + } else if (value == "g") { + event.scope = JsonEvent::Scope::kGlobal; + } else if (value.data() == nullptr) { + event.scope = JsonEvent::Scope::kNone; + } + } else if (it_.key() == "bp") { + event.bind_enclosing_slice = GetStringValue(it_.value()) == "e"; + } else if (it_.key() == "tts") { + event.tts = CoerceToTs(it_.value()); + } else if (it_.key() == "tdur") { + event.tdur = CoerceToTs(it_.value()); + } else if (it_.key() == "args") { + std::string_view args = GetObjectValue(it_.value()); + if (!args.empty()) { + event.args = std::make_unique(args.size()); + memcpy(event.args.get(), args.data(), args.size()); + event.args_size = args.size(); + } + } else if (it_.key() == "id2") { + std::string_view id2 = GetObjectValue(it_.value()); + if (!id2.empty()) { + ParseId2(id2, id2_local, id2_global); + } } - context_->sorter->PushJsonValue(ts, unparsed.ToStdString(), type); } - return SetOutAndReturn(next, out); + if (!event.phase) { + context_->storage->IncrementStats(stats::json_tokenizer_failure); + return true; + } + // Metadata events may omit ts. In all other cases error: + if (ts == std::numeric_limits::max() && event.phase != 'M') { + context_->storage->IncrementStats(stats::json_tokenizer_failure); + return true; + } + + // Make the tid equal to the pid if tid is not set. + if (event.tid == 0 && event.pid != 0 && !event.tid_is_string_id) { + event.tid = event.pid; + } + + if (PERFETTO_LIKELY(event.id_type == JsonEvent::IdType::kNone)) { + if (PERFETTO_UNLIKELY(!id2_global.empty())) { + event.async_cookie_type = JsonEvent::AsyncCookieType::kId2Global; + event.async_cookie = static_cast(base::Hasher::Combine( + event.cat.raw_id(), + base::StringView(id2_global.data(), id2_global.size()))); + } else if (PERFETTO_UNLIKELY(!id2_local.empty())) { + event.async_cookie_type = JsonEvent::AsyncCookieType::kId2Local; + event.async_cookie = static_cast(base::Hasher::Combine( + event.cat.raw_id(), + base::StringView(id2_local.data(), id2_local.size()))); + } + } else if (event.id_type == JsonEvent::IdType::kString) { + event.async_cookie_type = JsonEvent::AsyncCookieType::kId; + event.async_cookie = static_cast( + base::Hasher::Combine(event.cat.raw_id(), event.id.id_str.raw_id())); + } else if (event.id_type == JsonEvent::IdType::kUint64) { + event.async_cookie_type = JsonEvent::AsyncCookieType::kId; + event.async_cookie = static_cast( + base::Hasher::Combine(event.cat.raw_id(), event.id.id_uint64)); + } + if (PERFETTO_UNLIKELY(event.phase == 'P')) { + if (auto status = ParseV8SampleEvent(event); !status.ok()) { + context_->storage->IncrementStats(stats::json_tokenizer_failure); + return true; + } + return true; + } + context_->sorter->PushJsonValue(ts, std::move(event)); + return true; } -base::Status JsonTraceTokenizer::ParseV8SampleEvent(base::StringView unparsed) { - auto opt_evt = json::ParseJsonString(unparsed); - if (!opt_evt) { +void JsonTraceTokenizer::ParseId2(std::string_view id2, + std::string_view& id2_local, + std::string_view id2_global) { + inner_it_.Reset(id2.data(), id2.data() + id2.size()); + if (!inner_it_.ParseStart()) { + context_->storage->IncrementStats(stats::json_tokenizer_failure); + return; + } + for (;;) { + switch (inner_it_.ParseObjectFieldWithoutRecursing()) { + case State::kOk: + case State::kEndOfScope: + break; + case State::kError: + context_->storage->IncrementStats(stats::json_tokenizer_failure); + return; + case State::kIncompleteInput: + PERFETTO_FATAL("Unexpected incomplete input in JSON object for id2"); + } + if (inner_it_.eof()) { + break; + } + if (inner_it_.key() == "local") { + id2_local = GetStringValue(inner_it_.value()); + } else if (inner_it_.key() == "") { + id2_global = GetStringValue(inner_it_.value()); + } + } +} + +base::Status JsonTraceTokenizer::ParseV8SampleEvent(const JsonEvent& event) { + uint64_t id; + if (event.id_type == JsonEvent::IdType::kString) { + std::optional id_opt = base::StringToUInt64( + context_->storage->GetString(event.id.id_str).c_str(), 16); + if (!id_opt) { + context_->storage->IncrementStats(stats::json_tokenizer_failure); + return base::OkStatus(); + } + id = *id_opt; + } else if (event.id_type == JsonEvent::IdType::kUint64) { + id = event.id.id_uint64; + } else { return base::OkStatus(); } - const auto& evt = *opt_evt; - std::optional id = base::StringToUInt32(evt["id"].asString(), 16); - if (!id) { + auto args = json::ParseJsonString( + base::StringView(event.args.get(), event.args_size)); + if (!args) { return base::OkStatus(); } - uint32_t pid = evt["pid"].asUInt(); - uint32_t tid = evt["tid"].asUInt(); - const auto& val = evt["args"]["data"]; + const auto& val = (*args)["data"]; if (val.isMember("startTime")) { context_->legacy_v8_cpu_profile_tracker->SetStartTsForSessionAndPid( - *id, pid, val["startTime"].asInt64() * 1000); + id, event.pid, val["startTime"].asInt64() * 1000); return base::OkStatus(); } const auto& profile = val["cpuProfile"]; @@ -641,7 +702,7 @@ base::Status JsonTraceTokenizer::ParseV8SampleEvent(base::StringView unparsed) { frame.isMember("url") ? frame["url"].asCString() : base::StringView(); base::StringView function_name = frame["functionName"].asCString(); base::Status status = context_->legacy_v8_cpu_profile_tracker->AddCallsite( - *id, pid, node_id, parent_node_id, url, function_name, children); + id, event.pid, node_id, parent_node_id, url, function_name, children); if (!status.ok()) { context_->storage->IncrementStats( stats::legacy_v8_cpu_profile_invalid_callsite); @@ -657,8 +718,8 @@ base::Status JsonTraceTokenizer::ParseV8SampleEvent(base::StringView unparsed) { for (uint32_t i = 0; i < samples.size(); ++i) { ASSIGN_OR_RETURN(int64_t ts, context_->legacy_v8_cpu_profile_tracker->AddDeltaAndGetTs( - *id, pid, deltas[i].asInt64() * 1000)); - context_->sorter->PushLegacyV8CpuProfileEvent(ts, *id, pid, tid, + id, event.pid, deltas[i].asInt64() * 1000)); + context_->sorter->PushLegacyV8CpuProfileEvent(ts, id, event.pid, event.tid, samples[i].asUInt()); } return base::OkStatus(); @@ -726,7 +787,7 @@ base::Status JsonTraceTokenizer::HandleDictionaryKey(const char* start, return ParseInternal(next, end, out); } - // If we don't know the key for this JSON value just skip it. + // If we don't know the key for this JSON value just skip it_. switch (SkipOneJsonValue(next, end, &next)) { case SkipValueRes::kFatalError: return base::ErrStatus( diff --git a/src/trace_processor/importers/json/json_trace_tokenizer.h b/src/trace_processor/importers/json/json_trace_tokenizer.h index 398b67ddf1b..b5e31208855 100644 --- a/src/trace_processor/importers/json/json_trace_tokenizer.h +++ b/src/trace_processor/importers/json/json_trace_tokenizer.h @@ -18,42 +18,20 @@ #define SRC_TRACE_PROCESSOR_IMPORTERS_JSON_JSON_TRACE_TOKENIZER_H_ #include -#include #include +#include #include #include "perfetto/base/status.h" -#include "perfetto/ext/base/string_view.h" #include "src/trace_processor/importers/common/chunked_trace_reader.h" +#include "src/trace_processor/importers/common/parser_types.h" +#include "src/trace_processor/importers/json/json_parser.h" #include "src/trace_processor/importers/systrace/systrace_line_tokenizer.h" -namespace Json { -class Value; -} - namespace perfetto::trace_processor { class TraceProcessorContext; -// Visible for testing. -enum class ReadDictRes { - kFoundDict, - kNeedsMoreData, - kEndOfTrace, - kEndOfArray, -}; - -// Parses at most one JSON dictionary and returns a pointer to the end of it, -// or nullptr if no dict could be detected. -// This is to avoid decoding the full trace in memory and reduce heap traffic. -// E.g. input: { a:1 b:{ c:2, d:{ e:3 } } } , { a:4, ... }, -// output: [ only this is parsed ] ^return value points here. -// Visible for testing. -ReadDictRes ReadOneJsonDict(const char* start, - const char* end, - base::StringView* value, - const char** next); - enum class ReadKeyRes { kFoundKey, kNeedsMoreData, @@ -74,16 +52,6 @@ ReadKeyRes ReadOneJsonKey(const char* start, std::string* key, const char** next); -// Takes as input a JSON dictionary and returns the value associated with -// the provided key (if it exists). -// Implementation note: this method does not currently support dictionaries -// which have arrays as JSON values because current users of this method -// do not require this. -// Visible for testing. -base::Status ExtractValueForJsonKey(base::StringView dict, - const std::string& key, - std::optional* value); - enum class ReadSystemLineRes { kFoundLine, kNeedsMoreData, @@ -140,7 +108,11 @@ class JsonTraceTokenizer : public ChunkedTraceReader { const char* end, const char** out); - base::Status ParseV8SampleEvent(base::StringView unparsed); + bool ParseTraceEventContents(); + + void ParseId2(std::string_view, std::string_view&, std::string_view); + + base::Status ParseV8SampleEvent(const JsonEvent& event); base::Status HandleTraceEvent(const char* start, const char* end, @@ -160,6 +132,8 @@ class JsonTraceTokenizer : public ChunkedTraceReader { TracePosition position_ = TracePosition::kDictionaryKey; SystraceLineTokenizer systrace_line_tokenizer_; + json::Iterator it_; + json::Iterator inner_it_; uint64_t offset_ = 0; // Used to glue together JSON objects that span across two (or more) diff --git a/src/trace_processor/importers/json/json_trace_tokenizer_unittest.cc b/src/trace_processor/importers/json/json_trace_tokenizer_unittest.cc index 22a3885bc40..48fbe02c575 100644 --- a/src/trace_processor/importers/json/json_trace_tokenizer_unittest.cc +++ b/src/trace_processor/importers/json/json_trace_tokenizer_unittest.cc @@ -16,74 +16,15 @@ #include "src/trace_processor/importers/json/json_trace_tokenizer.h" -#include +#include +#include -#include "src/trace_processor/importers/json/json_utils.h" #include "test/gtest_and_gmock.h" -namespace perfetto { -namespace trace_processor { -namespace { - -TEST(JsonTraceTokenizerTest, ReadDictSuccess) { - const char* start = R"({ "foo": "bar" })"; - const char* end = start + strlen(start); - const char* next = nullptr; - base::StringView value; - ReadDictRes result = ReadOneJsonDict(start, end, &value, &next); - - ASSERT_EQ(result, ReadDictRes::kFoundDict); - ASSERT_EQ(next, end); - - Json::Value parsed = *json::ParseJsonString(value); - ASSERT_EQ(parsed["foo"].asString(), "bar"); -} - -TEST(JsonTraceTokenizerTest, ReadDictQuotedBraces) { - const char* start = R"({ "foo": "}\"bar{\\" })"; - const char* end = start + strlen(start); - const char* next = nullptr; - base::StringView value; - ReadDictRes result = ReadOneJsonDict(start, end, &value, &next); - - ASSERT_EQ(result, ReadDictRes::kFoundDict); - ASSERT_EQ(next, end); - - Json::Value parsed = *json::ParseJsonString(value); - ASSERT_EQ(parsed["foo"].asString(), "}\"bar{\\"); -} - -TEST(JsonTraceTokenizerTest, ReadDictTwoDicts) { - const char* start = R"({"foo": 1}, {"bar": 2})"; - const char* middle = start + strlen(R"({"foo": 1})"); - const char* end = start + strlen(start); - const char* next = nullptr; - base::StringView value; - - ASSERT_EQ(ReadOneJsonDict(start, end, &value, &next), - ReadDictRes::kFoundDict); - ASSERT_EQ(next, middle); - - Json::Value parsed = *json::ParseJsonString(value); - ASSERT_EQ(parsed["foo"].asInt(), 1); - - ASSERT_EQ(ReadOneJsonDict(next, end, &value, &next), ReadDictRes::kFoundDict); - ASSERT_EQ(next, end); - - parsed = *json::ParseJsonString(value); - ASSERT_EQ(parsed["bar"].asInt(), 2); -} - -TEST(JsonTraceTokenizerTest, ReadDictNeedMoreData) { - const char* start = R"({"foo": 1)"; - const char* end = start + strlen(start); - const char* next = nullptr; - base::StringView value; +#include - ASSERT_EQ(ReadOneJsonDict(start, end, &value, &next), - ReadDictRes::kNeedsMoreData); - ASSERT_EQ(next, nullptr); -} +namespace perfetto::trace_processor { +namespace { TEST(JsonTraceTokenizerTest, ReadKeyIntValue) { const char* start = R"("Test": 01234, )"; @@ -225,53 +166,5 @@ TEST(JsonTraceTokenizerTest, ReadSystraceEndOfData) { ASSERT_EQ(next, end); } -TEST(JsonTraceTokenizerTest, ExtractValueForJsonKey) { - std::optional line; - - ASSERT_TRUE(ExtractValueForJsonKey(R"({"ts": 149029})", "ts", &line).ok()); - ASSERT_EQ(*line, "149029"); - - ASSERT_TRUE(ExtractValueForJsonKey(R"( - { - "lots_of": "whitespace" - } - )", - "lots_of", &line) - .ok()); - ASSERT_EQ(*line, "whitespace"); - - ASSERT_TRUE(ExtractValueForJsonKey(R"( - { - "lots_of": "whitespace" , - "other": "value" - } - )", - "other", &line) - .ok()); - ASSERT_EQ(*line, "value"); - - ASSERT_TRUE(ExtractValueForJsonKey(R"({ - "ts": 149029, "foo": "bar" - })", - "ts", &line) - .ok()); - ASSERT_EQ(*line, "149029"); - - ASSERT_TRUE(ExtractValueForJsonKey(R"({ - "ts": 149029, "foo": "bar" - })", - "foo", &line) - .ok()); - ASSERT_EQ(*line, "bar"); - - ASSERT_TRUE(ExtractValueForJsonKey(R"({ - "nested": {"ts": 149029, "foo": "bar"} - })", - "nested", &line) - .ok()); - ASSERT_EQ(*line, R"({"ts": 149029, "foo": "bar"})"); -} - } // namespace -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor diff --git a/src/trace_processor/importers/json/json_utils.cc b/src/trace_processor/importers/json/json_utils.cc index d4157a17a3c..0bd521c3d89 100644 --- a/src/trace_processor/importers/json/json_utils.cc +++ b/src/trace_processor/importers/json/json_utils.cc @@ -17,127 +17,34 @@ #include "src/trace_processor/importers/json/json_utils.h" #include "perfetto/base/build_config.h" - -#include +#include "perfetto/base/compiler.h" +#include "perfetto/base/logging.h" +#include "perfetto/ext/base/small_vector.h" +#include "perfetto/ext/base/string_view.h" +#include "src/trace_processor/containers/string_pool.h" +#include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/json/json_parser.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/types/variadic.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include #if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) #include -#include "perfetto/ext/base/string_utils.h" -#endif - -namespace perfetto { -namespace trace_processor { -namespace json { - -std::optional CoerceToTs(const Json::Value& value) { - PERFETTO_DCHECK(IsJsonSupported()); - -#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) - switch (static_cast(value.type())) { - case Json::realValue: - return static_cast(value.asDouble() * 1000.0); - case Json::uintValue: - case Json::intValue: - return value.asInt64() * 1000; - case Json::stringValue: - return CoerceToTs(value.asString()); - default: - return std::nullopt; - } -#else - perfetto::base::ignore_result(value); - return std::nullopt; -#endif -} - -std::optional CoerceToTs(const std::string& s) { - PERFETTO_DCHECK(IsJsonSupported()); - -#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) - // 's' is formatted as a JSON Number, in microseconds - // goal: reformat 's' to be as an int, in nanoseconds - std::string s_as_ns = s; - - // detect and remove scientific notation's exponents - int32_t exp_shift = 0; - if (size_t exp_start = s.find_first_of("eE"); - exp_start != std::string::npos) { - const std::string exp_s = s.substr(exp_start + 1, s.size()); - const std::optional exp = base::StringToInt32(exp_s); - if (!exp.has_value()) { - return std::nullopt; - } - s_as_ns.erase(exp_start); - exp_shift = *exp; - } - - // detect and remove decimal separator - size_t int_size = s_as_ns.size(); - if (size_t frac_start = s.find('.'); frac_start != std::string::npos) { - s_as_ns.erase(frac_start, 1); - int_size = frac_start; - } - - // expand or shrink to the new size - constexpr int us_to_ns_shift = 3; - const size_t s_as_ns_size = size_t( - std::max(1, ptrdiff_t(int_size) + exp_shift + us_to_ns_shift)); - s_as_ns.resize(s_as_ns_size, '0'); // pads or truncates - - return base::StringToInt64(s_as_ns); -#else - perfetto::base::ignore_result(s); - return std::nullopt; +#include #endif -} -std::optional CoerceToInt64(const Json::Value& value) { - PERFETTO_DCHECK(IsJsonSupported()); - -#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) - switch (static_cast(value.type())) { - case Json::realValue: - case Json::uintValue: - return static_cast(value.asUInt64()); - case Json::intValue: - return value.asInt64(); - case Json::stringValue: { - std::string s = value.asString(); - char* end; - int64_t n = strtoll(s.c_str(), &end, 10); - if (end != s.data() + s.size()) - return std::nullopt; - return n; - } - default: - return std::nullopt; - } -#else - perfetto::base::ignore_result(value); - return std::nullopt; -#endif -} - -std::optional CoerceToUint32(const Json::Value& value) { - PERFETTO_DCHECK(IsJsonSupported()); - -#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) - std::optional result = CoerceToInt64(value); - if (!result.has_value()) - return std::nullopt; - int64_t n = result.value(); - if (n < 0 || n > std::numeric_limits::max()) - return std::nullopt; - return static_cast(n); -#else - perfetto::base::ignore_result(value); - return std::nullopt; -#endif -} +namespace perfetto::trace_processor::json { std::optional ParseJsonString(base::StringView raw_string) { - PERFETTO_DCHECK(IsJsonSupported()); - #if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) Json::CharReaderBuilder b; auto reader = std::unique_ptr(b.newCharReader()); @@ -153,87 +60,112 @@ std::optional ParseJsonString(base::StringView raw_string) { #endif } -bool AddJsonValueToArgs(const Json::Value& value, - base::StringView flat_key, - base::StringView key, +bool AddJsonValueToArgs(Iterator& it, + const char* start, + const char* end, + std::string_view flat_key, + std::string_view key, TraceStorage* storage, ArgsTracker::BoundInserter* inserter) { - PERFETTO_DCHECK(IsJsonSupported()); - -#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) - if (value.isObject()) { - auto it = value.begin(); + it.Reset(start, end); + if (!it.ParseStart()) { + return false; + } + struct Frame { + uint32_t flat_key_size = 0; + uint32_t key_size = 0; bool inserted = false; - for (; it != value.end(); ++it) { - std::string child_name = it.name(); - std::string child_flat_key = flat_key.ToStdString() + "." + child_name; - std::string child_key = key.ToStdString() + "." + child_name; - inserted |= - AddJsonValueToArgs(*it, base::StringView(child_flat_key), - base::StringView(child_key), storage, inserter); + }; + base::SmallVector stack; + std::string flat_key_str(flat_key); + std::string key_str(key); + + stack.emplace_back(Frame{static_cast(flat_key_str.size()), + static_cast(key_str.size()), false}); + for (;;) { + json::Iterator::ParseType parse_type = it.parse_stack().back(); + auto& frame = stack.back(); + switch (it.ParseAndRecurse()) { + case json::ReturnCode::kOk: + break; + case json::ReturnCode::kEndOfScope: { + PERFETTO_DCHECK(!stack.empty()); + bool inserted = frame.inserted; + stack.pop_back(); + if (stack.empty()) { + return inserted; + } + auto& new_frame = stack.back(); + new_frame.inserted |= inserted; + continue; + } + case json::ReturnCode::kIncompleteInput: + case json::ReturnCode::kError: + return false; } - return inserted; - } - if (value.isArray()) { - auto it = value.begin(); - bool inserted_any = false; - std::string array_key = key.ToStdString(); - StringId array_key_id = storage->InternString(key); - for (; it != value.end(); ++it) { - size_t array_index = inserter->GetNextArrayEntryIndex(array_key_id); - std::string child_key = - array_key + "[" + std::to_string(array_index) + "]"; - bool inserted = AddJsonValueToArgs( - *it, flat_key, base::StringView(child_key), storage, inserter); - if (inserted) - inserter->IncrementArrayEntryIndex(array_key_id); - inserted_any |= inserted; + // Skip null values. + const auto& value = it.value(); + if (value.index() == base::variant_index()) { + continue; + } + + // Leaf value. + flat_key_str.resize(frame.flat_key_size); + key_str.resize(frame.key_size); + + if (parse_type == json::Iterator::ParseType::kArray) { + StringPool::Id key_id = storage->InternString(key_str); + size_t array_index = inserter->GetNextArrayEntryIndex(key_id); + key_str += "[" + std::to_string(array_index) + "]"; + inserter->IncrementArrayEntryIndex(key_id); + } else if (parse_type == json::Iterator::ParseType::kObject) { + key_str += "."; + key_str += it.key(); + flat_key_str += "."; + flat_key_str += it.key(); + } else { + PERFETTO_FATAL("Unexpected parse type %d", static_cast(parse_type)); + } + + if (value.index() == base::variant_index() || + value.index() == base::variant_index()) { + stack.emplace_back(Frame{static_cast(flat_key_str.size()), + static_cast(key_str.size()), false}); + continue; } - return inserted_any; - } - // Leaf value. - auto flat_key_id = storage->InternString(flat_key); - auto key_id = storage->InternString(key); - - switch (value.type()) { - case Json::ValueType::nullValue: - break; - case Json::ValueType::intValue: - inserter->AddArg(flat_key_id, key_id, Variadic::Integer(value.asInt64())); - return true; - case Json::ValueType::uintValue: - inserter->AddArg(flat_key_id, key_id, - Variadic::UnsignedInteger(value.asUInt64())); - return true; - case Json::ValueType::realValue: - inserter->AddArg(flat_key_id, key_id, Variadic::Real(value.asDouble())); - return true; - case Json::ValueType::stringValue: - inserter->AddArg(flat_key_id, key_id, - Variadic::String(storage->InternString( - base::StringView(value.asString())))); - return true; - case Json::ValueType::booleanValue: - inserter->AddArg(flat_key_id, key_id, Variadic::Boolean(value.asBool())); - return true; - case Json::ValueType::objectValue: - case Json::ValueType::arrayValue: - PERFETTO_FATAL("Non-leaf types handled above"); - break; + // Only for leaf values we actually insert into the args table. + frame.inserted = true; + + StringPool::Id flat_key_id = storage->InternString(flat_key_str); + StringPool::Id key_id = storage->InternString(key_str); + switch (it.value().index()) { + case base::variant_index(): + inserter->AddArg( + flat_key_id, key_id, + Variadic::Real(base::unchecked_get(it.value()))); + break; + case base::variant_index(): + inserter->AddArg( + flat_key_id, key_id, + Variadic::Integer(base::unchecked_get(it.value()))); + break; + case base::variant_index(): + inserter->AddArg( + flat_key_id, key_id, + Variadic::String(storage->InternString( + base::unchecked_get(it.value())))); + break; + case base::variant_index(): + inserter->AddArg( + flat_key_id, key_id, + Variadic::Boolean(base::unchecked_get(it.value()))); + break; + default: + PERFETTO_FATAL("Unreachable"); + } } - return false; -#else - perfetto::base::ignore_result(value); - perfetto::base::ignore_result(flat_key); - perfetto::base::ignore_result(key); - perfetto::base::ignore_result(storage); - perfetto::base::ignore_result(inserter); - return false; -#endif } -} // namespace json -} // namespace trace_processor -} // namespace perfetto +} // namespace perfetto::trace_processor::json diff --git a/src/trace_processor/importers/json/json_utils.h b/src/trace_processor/importers/json/json_utils.h index 3c1d8faebab..ccc3d8e341f 100644 --- a/src/trace_processor/importers/json/json_utils.h +++ b/src/trace_processor/importers/json/json_utils.h @@ -17,13 +17,13 @@ #ifndef SRC_TRACE_PROCESSOR_IMPORTERS_JSON_JSON_UTILS_H_ #define SRC_TRACE_PROCESSOR_IMPORTERS_JSON_JSON_UTILS_H_ -#include #include -#include +#include #include "perfetto/base/build_config.h" #include "perfetto/ext/base/string_view.h" #include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/json/json_parser.h" #include "src/trace_processor/storage/trace_storage.h" #if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) @@ -46,24 +46,22 @@ constexpr bool IsJsonSupported() { #endif } -std::optional CoerceToTs(const Json::Value& value); -std::optional CoerceToTs(const std::string& value); -std::optional CoerceToInt64(const Json::Value& value); -std::optional CoerceToUint32(const Json::Value& value); - // Parses the given JSON string into a JSON::Value object. // This function should only be called if |IsJsonSupported()| returns true. std::optional ParseJsonString(base::StringView raw_string); -// Flattens the given Json::Value and adds each leaf node to the bound args -// inserter. Note: +// Flattens the given serialized json value (with bounds `start` and `end`) +// using `it` and adds each leaf node to the bound args inserter. +// +// Note: // * |flat_key| and |key| should be non-empty and will be used to prefix the // keys of all leaf nodes in the JSON. // * |storage| is used to intern all strings (e.g. keys and values). -// * This function should only be called if |IsJsonSupported()| returns true. -bool AddJsonValueToArgs(const Json::Value& value, - base::StringView flat_key, - base::StringView key, +bool AddJsonValueToArgs(Iterator& it, + const char* start, + const char* end, + std::string_view flat_key, + std::string_view key, TraceStorage* storage, ArgsTracker::BoundInserter* inserter); diff --git a/src/trace_processor/importers/json/json_utils_unittest.cc b/src/trace_processor/importers/json/json_utils_unittest.cc index 3d9be2746db..db7dcf32680 100644 --- a/src/trace_processor/importers/json/json_utils_unittest.cc +++ b/src/trace_processor/importers/json/json_utils_unittest.cc @@ -14,67 +14,9 @@ * limitations under the License. */ -#include "src/trace_processor/importers/json/json_utils.h" - #include #include -#include "test/gtest_and_gmock.h" - namespace perfetto::trace_processor::json { -namespace { - -TEST(JsonTraceUtilsTest, CoerceToUint32) { - ASSERT_EQ(CoerceToUint32(Json::Value(42)).value_or(0), 42u); - ASSERT_EQ(CoerceToUint32(Json::Value("42")).value_or(0), 42u); - ASSERT_EQ(CoerceToInt64(Json::Value(42.1)).value_or(-1), 42); -} - -TEST(JsonTraceUtilsTest, CoerceToInt64) { - ASSERT_EQ(CoerceToInt64(Json::Value(42)).value_or(-1), 42); - ASSERT_EQ(CoerceToInt64(Json::Value("42")).value_or(-1), 42); - ASSERT_EQ(CoerceToInt64(Json::Value(42.1)).value_or(-1), 42); - ASSERT_FALSE(CoerceToInt64(Json::Value("foo")).has_value()); - ASSERT_FALSE(CoerceToInt64(Json::Value("1234!")).has_value()); - - Json::UInt64 n = 18446744073709551615UL; - ASSERT_EQ(CoerceToInt64(Json::Value{n}).value_or(0), -1); -} - -TEST(JsonTraceUtilsTest, CoerceToTs) { - ASSERT_EQ(CoerceToTs(Json::Value(42)).value_or(-1), 42000); - ASSERT_EQ(CoerceToTs(Json::Value("42")).value_or(-1), 42000); - ASSERT_EQ(CoerceToTs(Json::Value(42.1)).value_or(-1), 42100); - ASSERT_EQ(CoerceToTs(Json::Value("42.1")).value_or(-1), 42100); - ASSERT_EQ(CoerceToTs(Json::Value(".42")).value_or(-1), 420); - ASSERT_EQ(CoerceToTs(Json::Value("42.")).value_or(-1), 42000); - ASSERT_EQ(CoerceToTs(Json::Value("42.0")).value_or(-1), 42000); - ASSERT_EQ(CoerceToTs(Json::Value("0.2")).value_or(-1), 200); - ASSERT_EQ(CoerceToTs(Json::Value("0.2e-1")).value_or(-1), 20); - ASSERT_EQ(CoerceToTs(Json::Value("0.2e-2")).value_or(-1), 2); - ASSERT_EQ(CoerceToTs(Json::Value("0.2e-3")).value_or(-1), 0); - ASSERT_EQ(CoerceToTs(Json::Value("1.692108548132154500e+15")).value_or(-1), - 1'692'108'548'132'154'500); - ASSERT_EQ(CoerceToTs(Json::Value("1692108548132154.500")).value_or(-1), - 1'692'108'548'132'154'500); - ASSERT_EQ(CoerceToTs(Json::Value("1.692108548132154501e+15")).value_or(-1), - 1'692'108'548'132'154'501); - ASSERT_EQ(CoerceToTs(Json::Value("1692108548132154.501")).value_or(-1), - 1'692'108'548'132'154'501); - ASSERT_EQ(CoerceToTs(Json::Value("-1.692108548132154500E+15")).value_or(-1), - -1'692'108'548'132'154'500); - ASSERT_EQ(CoerceToTs(Json::Value("-1692108548132154.500")).value_or(-1), - -1'692'108'548'132'154'500); - ASSERT_EQ(CoerceToTs(Json::Value("-1.692108548132154501E+15")).value_or(-1), - -1'692'108'548'132'154'501); - ASSERT_EQ(CoerceToTs(Json::Value("-1692108548132154.501")).value_or(-1), - -1'692'108'548'132'154'501); - ASSERT_EQ(CoerceToTs(Json::Value("-0")).value_or(-1), 0); - ASSERT_EQ(CoerceToTs(Json::Value("0")).value_or(-1), 0); - ASSERT_EQ(CoerceToTs(Json::Value(".")).value_or(-1), 0); - ASSERT_FALSE(CoerceToTs(Json::Value("1234!")).has_value()); - ASSERT_FALSE(CoerceToTs(Json::Value("123e4!")).has_value()); -} - -} // namespace +namespace {} // namespace } // namespace perfetto::trace_processor::json diff --git a/src/trace_processor/importers/proto/args_parser.cc b/src/trace_processor/importers/proto/args_parser.cc index 30801903c95..bbeed046a0f 100644 --- a/src/trace_processor/importers/proto/args_parser.cc +++ b/src/trace_processor/importers/proto/args_parser.cc @@ -16,8 +16,21 @@ #include "src/trace_processor/importers/proto/args_parser.h" +#include +#include +#include + +#include "perfetto/base/logging.h" #include "perfetto/ext/base/base64.h" +#include "perfetto/ext/base/string_view.h" +#include "perfetto/protozero/field.h" +#include "src/trace_processor/importers/common/args_tracker.h" +#include "src/trace_processor/importers/json/json_parser.h" #include "src/trace_processor/importers/json/json_utils.h" +#include "src/trace_processor/importers/proto/packet_sequence_state_generation.h" +#include "src/trace_processor/storage/trace_storage.h" +#include "src/trace_processor/types/variadic.h" +#include "src/trace_processor/util/interned_message_view.h" namespace perfetto::trace_processor { @@ -91,9 +104,9 @@ bool ArgsParser::AddJson(const Key& key, const protozero::ConstChars& value) { auto json_value = json::ParseJsonString(value); if (!json_value) return false; - return json::AddJsonValueToArgs(*json_value, base::StringView(key.flat_key), - base::StringView(key.key), &storage_, - &inserter_); + json::Iterator iterator; + return json::AddJsonValueToArgs(iterator, value.data, value.data + value.size, + key.flat_key, key.key, &storage_, &inserter_); } void ArgsParser::AddNull(const Key& key) { diff --git a/src/trace_processor/sorter/trace_sorter.cc b/src/trace_processor/sorter/trace_sorter.cc index e6dfe106fd0..c342a380d0c 100644 --- a/src/trace_processor/sorter/trace_sorter.cc +++ b/src/trace_processor/sorter/trace_sorter.cc @@ -260,7 +260,7 @@ void TraceSorter::ParseTracePacket(TraceProcessorContext& context, return; case TimestampedEvent::Type::kJsonValue: context.json_trace_parser->ParseJsonPacket( - event.ts, std::move(token_buffer_.Extract(id).value)); + event.ts, token_buffer_.Extract(id)); return; case TimestampedEvent::Type::kSpeRecord: context.spe_record_parser->ParseSpeRecord( diff --git a/src/trace_processor/sorter/trace_sorter.h b/src/trace_processor/sorter/trace_sorter.h index 18a7e362292..351865d6a87 100644 --- a/src/trace_processor/sorter/trace_sorter.h +++ b/src/trace_processor/sorter/trace_sorter.h @@ -190,17 +190,15 @@ class TraceSorter { machine_id); } - void PushJsonValue(int64_t timestamp, - std::string json_value, - const JsonEvent::Type& type) { - if (const auto* scoped = std::get_if(&type); scoped) { + void PushJsonValue(int64_t timestamp, JsonEvent json_value) { + if (json_value.phase == 'X') { // We need to account for slices with duration by sorting them specially: // this requires us to use the slower comparator which takes this into // account. use_slow_sorting_ = true; } AppendNonFtraceEvent(timestamp, TimestampedEvent::Type::kJsonValue, - JsonEvent{std::move(json_value), type}); + std::move(json_value)); } void PushFuchsiaRecord(int64_t timestamp, FuchsiaRecord fuchsia_record) { @@ -410,21 +408,18 @@ class TraceSorter { // For std::sort() in slow mode. bool operator()(const TimestampedEvent& a, const TimestampedEvent& b) const { - int64_t a_key = - KeyForType(buffer.Get(GetTokenBufferId(a))->type); - int64_t b_key = - KeyForType(buffer.Get(GetTokenBufferId(b))->type); + int64_t a_key = KeyForType(*buffer.Get(GetTokenBufferId(a))); + int64_t b_key = KeyForType(*buffer.Get(GetTokenBufferId(b))); return std::tie(a.ts, a_key, a.chunk_index, a.chunk_offset) < std::tie(b.ts, b_key, b.chunk_index, b.chunk_offset); } - static int64_t KeyForType(const JsonEvent::Type& type) { - switch (type.index()) { - case base::variant_index(): + static int64_t KeyForType(const JsonEvent& event) { + switch (event.phase) { + case 'E': return std::numeric_limits::min(); - case base::variant_index(): - return std::numeric_limits::max() - - std::get(type).dur; + case 'X': + return std::numeric_limits::max() - event.dur; default: return std::numeric_limits::max(); } diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc index a79695237df..a513d872167 100644 --- a/src/trace_processor/trace_processor_impl.cc +++ b/src/trace_processor/trace_processor_impl.cc @@ -452,12 +452,11 @@ TraceProcessorImpl::TraceProcessorImpl(const Config& cfg) context_.reader_registry->RegisterTraceReader(kZipFile); } - if constexpr (json::IsJsonSupported()) { - context_.reader_registry->RegisterTraceReader( - kJsonTraceType); - context_.json_trace_parser = - std::make_unique(&context_); + context_.reader_registry->RegisterTraceReader( + kJsonTraceType); + context_.json_trace_parser = std::make_unique(&context_); + if constexpr (json::IsJsonSupported()) { context_.reader_registry ->RegisterTraceReader( kGeckoTraceType); diff --git a/src/trace_processor/util/debug_annotation_parser.cc b/src/trace_processor/util/debug_annotation_parser.cc index 92f8d4baf78..fe57c5b4ca2 100644 --- a/src/trace_processor/util/debug_annotation_parser.cc +++ b/src/trace_processor/util/debug_annotation_parser.cc @@ -43,14 +43,6 @@ std::string SanitizeDebugAnnotationName(const std::string& raw_name) { return result; } -constexpr bool IsJsonSupported() { -#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON) - return true; -#else - return false; -#endif -} - } // namespace DebugAnnotationParser::DebugAnnotationParser(ProtoToArgsParser& parser) @@ -138,10 +130,6 @@ DebugAnnotationParser::ParseDebugAnnotationValue( return {value_parse_result.status, added_entry}; } } else if (annotation.has_legacy_json_value()) { - if (!IsJsonSupported()) - return {base::ErrStatus("Ignoring legacy_json_value (no json support)"), - false}; - bool added_entry = delegate.AddJson(context_name, annotation.legacy_json_value()); return {base::OkStatus(), added_entry}; diff --git a/test/trace_processor/diff_tests/parser/json/tests.py b/test/trace_processor/diff_tests/parser/json/tests.py index ba19bc193bb..38fce16e392 100644 --- a/test/trace_processor/diff_tests/parser/json/tests.py +++ b/test/trace_processor/diff_tests/parser/json/tests.py @@ -89,7 +89,7 @@ def test_args_ordered(self): "01.step1": 1, "02.step2": 2, } - }, + } ]'''), query=''' SELECT @@ -149,3 +149,629 @@ def test_json_incomplete(self): "ts" 4619295550000 ''')) + + def test_json_all_slice_types(self): + return DiffTestBlueprint( + trace=Json(''' + [ + { + "ph": "B", + "pid": 10, + "tid": 11, + "ts": 100, + "name": "BeginEvent", + "cat": "cat1", + "args": { "arg_str": "hello", "arg_int": 123, "arg_bool": true } + }, + { + "ph": "E", + "pid": 10, + "tid": 11, + "ts": 200, + "name": "EndEvent", + "cat": "cat1", + "tts": 190, + "args": { "arg_double": 45.67 } + }, + { + "ph": "X", + "pid": 10, + "tid": 12, + "ts": 250, + "dur": 50, + "name": "CompleteEvent", + "cat": "cat2,cat3", + "tts": 240, + "tdur": 40, + "args": { "arg_null": null, "another_int": -5 } + }, + { + "ph": "I", + "pid": 10, + "tid": 11, + "ts": 300, + "name": "InstantGlobal", + "cat": "cat_inst", + "s": "g", + "args": {} + }, + { + "ph": "I", + "pid": 10, + "tid": 0, + "ts": 310, + "name": "InstantProcess", + "cat": "cat_inst", + "s": "p", + "args": { "scope_val": "process" } + }, + { + "ph": "I", + "pid": 10, + "tid": 11, + "ts": 320, + "name": "InstantThread", + "cat": "cat_inst", + "s": "t" + }, + { + "ph": "I", + "pid": 10, + "tid": 12, + "ts": 330, + "name": "InstantDefaultScope", + "cat": "cat_inst" + }, + { + "ph": "R", + "pid": 10, + "tid": 11, + "ts": 340, + "name": "InstantR", + "cat": "cat_inst", + "s": "g" + }, + { + "ph": "i", + "pid": 10, + "tid": 11, + "ts": 350, + "name": "Instanti", + "cat": "cat_inst", + "s": "t" + } + ] + '''), + query=''' + INCLUDE PERFETTO MODULE slices.with_context; + + SELECT + slice.ts, + slice.dur, + slice.name, + args.flat_key, + args.key, + args.string_value, + args.int_value + FROM thread_or_process_slice AS slice + LEFT JOIN args ON slice.arg_set_id = args.arg_set_id + ''', + out=Csv(""" + "ts","dur","name","flat_key","key","string_value","int_value" + 100000,-1,"BeginEvent","args.arg_bool","args.arg_bool","[NULL]",1 + 100000,-1,"BeginEvent","args.arg_int","args.arg_int","[NULL]",123 + 100000,-1,"BeginEvent","args.arg_str","args.arg_str","hello","[NULL]" + 320000,0,"InstantThread","[NULL]","[NULL]","[NULL]","[NULL]" + 350000,0,"Instanti","[NULL]","[NULL]","[NULL]","[NULL]" + 250000,50000,"CompleteEvent","args.another_int","args.another_int","[NULL]",-5 + 330000,0,"InstantDefaultScope","[NULL]","[NULL]","[NULL]","[NULL]" + 310000,0,"InstantProcess","args.scope_val","args.scope_val","process","[NULL]" + """)) + + def test_json_flow(self): + return DiffTestBlueprint( + trace=Json(''' + [ + { + "ph": "B", + "pid": 10, + "tid": 11, + "ts": 100, + "name": "BeginEvent", + "cat": "cat1", + "args": { "arg_str": "hello", "arg_int": 123, "arg_bool": true } + }, + { + "ph": "E", + "pid": 10, + "tid": 11, + "ts": 200, + "name": "EndEvent", + "cat": "cat1", + "tts": 190, + "args": { "arg_double": 45.67 } + }, + { + "ph": "X", + "pid": 10, + "tid": 12, + "ts": 250, + "dur": 50, + "name": "CompleteEvent", + "cat": "cat2,cat3", + "tts": 240, + "tdur": 40, + "args": { "arg_null": null, "another_int": -5 } + }, + { + "ph": "I", + "pid": 10, + "tid": 11, + "ts": 300, + "name": "InstantGlobal", + "cat": "cat_inst", + "s": "g", + "args": {} + }, + { + "ph": "I", + "pid": 10, + "tid": 0, + "ts": 310, + "name": "InstantProcess", + "cat": "cat_inst", + "s": "p", + "args": { "scope_val": "process" } + }, + { + "ph": "I", + "pid": 10, + "tid": 11, + "ts": 320, + "name": "InstantThread", + "cat": "cat_inst", + "s": "t" + }, + { + "ph": "I", + "pid": 10, + "tid": 12, + "ts": 330, + "name": "InstantDefaultScope", + "cat": "cat_inst" + }, + { + "ph": "R", + "pid": 10, + "tid": 11, + "ts": 340, + "name": "InstantR", + "cat": "cat_inst", + "s": "g" + }, + { + "ph": "i", + "pid": 10, + "tid": 11, + "ts": 350, + "name": "Instanti", + "cat": "cat_inst", + "s": "t" + } + ] + '''), + query=''' + INCLUDE PERFETTO MODULE slices.with_context; + + SELECT + slice.ts, + slice.dur, + slice.name, + COUNT(DISTINCT fin.id) AS fin, + COUNT(DISTINCT fout.id) AS fout + FROM thread_or_process_slice AS slice + LEFT JOIN flow as fin ON slice.id = fin.slice_in + LEFT JOIN flow as fout ON slice.id = fout.slice_out + ''', + out=Csv(""" + "ts","dur","name","fin","fout" + 100000,-1,"BeginEvent",0,0 + """)) + + def test_json_metadata(self): + return DiffTestBlueprint( + trace=Json('''[ + { + "ph": "B", + "pid": 30, + "tid": 31, + "ts": 790, + "name": "ActivityOnThread31", + "cat": "test_cat_setup" + }, + { + "ph": "C", + "pid": 30, + "tid": 31, + "ts": 800, + "name": "MyCounters", + "cat": "stats", + "id": "counter_group_1", + "args": { + "counter1_float": 100.5, + "counter2_int": -50, + "counter3_str_num": "25.7" + } + }, + { + "ph": "E", + "pid": 30, + "tid": 31, + "ts": 815, + "name": "ActivityOnThread31", + "cat": "test_cat_setup" + }, + { + "ph": "C", + "pid": 30, + "tid": 31, + "ts": 810, + "name": "EmptyArgsCounter", + "cat": "stats", + "args": {} + }, + { + "ph": "C", + "pid": 30, + "tid": 31, + "ts": 820, + "name": "NoArgsCounter", + "cat": "stats" + }, + { + "ph": "M", + "pid": 30, + "tid": 0, + "ts": 830, + "name": "process_name", + "cat": "__metadata", + "args": { "name": "MyProcess30" } + }, + { + "ph": "M", + "pid": 30, + "tid": 31, + "ts": 840, + "name": "thread_name", + "cat": "__metadata", + "args": { "name": "MyRevisedThread31Name" } + }, + { + "ph": "M", + "pid": 30, + "tid": 0, + "ts": 850, + "name": "process_sort_index", + "cat": "__metadata", + "args": { "sort_index": -2 } + }, + { + "ph": "M", + "pid": 30, + "tid": 31, + "ts": 860, + "name": "thread_sort_index", + "cat": "__metadata", + "args": { "sort_index": 5 } + }, + { + "ph": "B", + "pid": 30, + "tid": 32, + "ts": 900, + "name": "EventMissingCatAndArgs" + }, + { + "ph": "E", + "pid": 30, + "tid": 32, + "ts": 950 + }, + { + "ph": "X", + "pid": 33, + "ts": 960, + "dur": 10, + "name": "PidOnlyEvent", + "cat": "special_pids" + }, + { + "ph": "X", + "ts": 970, + "dur": 10, + "name": "NoPidNoTidEvent", + "cat": "special_pids" + } + ] + + '''), + query=''' + INCLUDE PERFETTO MODULE slices.with_context; + + SELECT + slice.ts, + slice.dur, + slice.name, + slice.thread_name, + slice.tid, + slice.process_name, + slice.pid, + args.flat_key, + args.key, + args.string_value, + args.int_value + FROM thread_or_process_slice AS slice + LEFT JOIN args ON slice.arg_set_id = args.arg_set_id + ''', + out=Csv(""" + "ts","dur","name","thread_name","tid","process_name","pid","flat_key","key","string_value","int_value" + 790000,25000,"ActivityOnThread31","MyRevisedThread31Name",31,"MyProcess30",30,"[NULL]","[NULL]","[NULL]","[NULL]" + 900000,50000,"EventMissingCatAndArgs","[NULL]",32,"MyProcess30",30,"[NULL]","[NULL]","[NULL]","[NULL]" + 960000,10000,"PidOnlyEvent","[NULL]",33,"[NULL]",33,"[NULL]","[NULL]","[NULL]","[NULL]" + 970000,10000,"NoPidNoTidEvent","[NULL]",0,"[NULL]",0,"[NULL]","[NULL]","[NULL]","[NULL]" + """)) + + def test_json_extreme_vals(self): + return DiffTestBlueprint( + trace=Json(''' + [ + { + "ph": "X", + "pid": 100, + "tid": 101, + "ts": 1000, + "name": "CompleteNoDur", + "cat": "test_missing" + }, + { + "ph": "B", + "pid": 100, + "tid": 102, + "ts": 1100, + "name": "BeginNoTTS", + "cat": "test_missing" + }, + { + "ph": "E", + "pid": 100, + "tid": 102, + "ts": 1200, + "name": "EndNoTTS", + "cat": "test_missing" + }, + { + "ph": "X", + "pid": 100, + "tid": 103, + "ts": 1300, + "dur": 50, + "name": "CompleteNoTDUR", + "cat": "test_missing", + "tts": 1290 + } + ] + '''), + query=''' + INCLUDE PERFETTO MODULE slices.with_context; + + SELECT + slice.ts, + slice.dur, + slice.name, + slice.thread_name, + slice.tid, + slice.process_name, + slice.pid, + args.flat_key, + args.key, + args.string_value, + args.int_value + FROM thread_or_process_slice AS slice + LEFT JOIN args ON slice.arg_set_id = args.arg_set_id + ''', + out=Csv(""" + "ts","dur","name","thread_name","tid","process_name","pid","flat_key","key","string_value","int_value" + 1100000,-1,"BeginNoTTS","[NULL]",102,"[NULL]",100,"[NULL]","[NULL]","[NULL]","[NULL]" + 1300000,50000,"CompleteNoTDUR","[NULL]",103,"[NULL]",100,"[NULL]","[NULL]","[NULL]","[NULL]" + """)) + + def test_json_async(self): + return DiffTestBlueprint( + trace=Json('''[ + { + "ph": "b", + "pid": 200, + "tid": 201, + "ts": 2000, + "name": "AsyncMissingCat", + "id": "flow1" + }, + { + "ph": "e", + "pid": 200, + "tid": 201, + "ts": 2100, + "name": "AsyncMissingCat", + "id": "flow1" + }, + { + "ph": "b", + "pid": 200, + "tid": 202, + "ts": 2200, + "name": "AsyncEmptyStringId", + "cat": "category2", + "id": "" + }, + { + "ph": "e", + "pid": 200, + "tid": 202, + "ts": 2300, + "name": "AsyncEmptyStringId", + "cat": "category2", + "id": "" + }, + { + "ph": "b", + "pid": 200, + "tid": 203, + "ts": 2400, + "name": "AsyncMissingCatEmptyStringId", + "id": "" + }, + { + "ph": "e", + "pid": 200, + "tid": 203, + "ts": 2500, + "name": "AsyncMissingCatEmptyStringId", + "id": "" + } + ] + '''), + query=''' + INCLUDE PERFETTO MODULE slices.with_context; + + SELECT + slice.ts, + slice.dur, + slice.name, + slice.thread_name, + slice.tid, + slice.process_name, + slice.pid, + args.flat_key, + args.key, + args.string_value, + args.int_value + FROM thread_or_process_slice AS slice + LEFT JOIN args ON slice.arg_set_id = args.arg_set_id + ''', + out=Csv(""" + "ts","dur","name","thread_name","tid","process_name","pid","flat_key","key","string_value","int_value" + """)) + + def test_json_counter_args(self): + return DiffTestBlueprint( + trace=Json(''' + [ + { + "ph": "C", + "pid": 300, + "tid": 301, + "ts": 3000, + "name": "CounterStringEdgeCases", + "cat": "counters_str", + "args": { + "leading_dot": ".75", + "trailing_dot": "246.", + "leading_zero_float": "01.23", + "leading_zero_int_str": "050", + "incomplete_exp": "1.5e", + "exp_no_digit": "2e+", + "just_dot": ".", + "plus_val": "+10.5", + "just_plus": "+", + "just_minus": "-" + } + } + ] + '''), + query=''' + SELECT process.pid, process.name, pct.name, value + FROM counter + JOIN process_counter_track pct ON counter.track_id = pct.id + JOIN process USING (upid) + ORDER BY pct.name + ''', + out=Csv(""" + "pid","name","name","value" + 300,"[NULL]","CounterStringEdgeCases leading_dot",0.750000 + 300,"[NULL]","CounterStringEdgeCases leading_zero_float",1.230000 + 300,"[NULL]","CounterStringEdgeCases leading_zero_int_str",50.000000 + 300,"[NULL]","CounterStringEdgeCases plus_val",10.500000 + 300,"[NULL]","CounterStringEdgeCases trailing_dot",246.000000 + """)) + + def test_json_id2(self): + return DiffTestBlueprint( + trace=Json(''' + [ + { + "ph": "b", + "pid": 400, + "tid": 401, + "ts": 4000, + "name": "AsyncId2StandardGlobalKey", + "cat": "id2test", + "id2": { + "global": "standard_global_id_1" + } + }, + { + "ph": "e", + "pid": 400, + "tid": 401, + "ts": 4100, + "name": "AsyncId2StandardGlobalKey", + "cat": "id2test", + "id2": { + "global": "standard_global_id_1" + } + }, + { + "ph": "b", + "pid": 400, + "tid": 402, + "ts": 4200, + "name": "AsyncId2EmptyGlobalKey", + "cat": "id2test", + "id2": { + "": "empty_key_global_id_2" + } + }, + { + "ph": "e", + "pid": 400, + "tid": 402, + "ts": 4300, + "name": "AsyncId2EmptyGlobalKey", + "cat": "id2test", + "id2": { + "": "empty_key_global_id_2" + } + } + ] + '''), + query=''' + INCLUDE PERFETTO MODULE slices.with_context; + + SELECT + slice.ts, + slice.dur, + slice.name, + slice.thread_name, + slice.tid, + slice.process_name, + slice.pid, + args.flat_key, + args.key, + args.string_value, + args.int_value + FROM thread_or_process_slice AS slice + LEFT JOIN args ON slice.arg_set_id = args.arg_set_id + ''', + out=Csv(""" + "ts","dur","name","thread_name","tid","process_name","pid","flat_key","key","string_value","int_value" + """))