Skip to content

Commit ce13c00

Browse files
staticfloatstaticfloat
committed
tp: Parse JournaldEventPacket into unified LogTable
Refactor `AndroidLogTable` into a unified `LogTable` (`log_tables.py`, exposed as `__intrinsic_logs`) that stores log entries from all sources. A new log_source STRING column identifies the origin: 'android' or 'journald'. `LogTable` is moved out of `android_tables.py` into its own `log_tables.py` since it is no longer Android-specific. New linux_probes_parser.cc parses `JournaldEventPacket` and inserts into `LogTable` with log_source='journald'. Journald-specific metadata (uid, comm, systemd_unit, hostname, transport) is stored via `ArgsTracker`. Three SQL views are provided in the prelude: - logs: all entries from all sources - android_logs: convenience alias for log_source='android' entries - journald_logs: journald entries with metadata columns via extract_arg() Diff tests updated and a new Linux diff test suite covers journald parsing.
1 parent aaf3540 commit ce13c00

23 files changed

Lines changed: 623 additions & 67 deletions

‎src/trace_processor/importers/android_bugreport/android_log_event_parser.cc‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,29 @@
1717
#include "src/trace_processor/importers/android_bugreport/android_log_event_parser.h"
1818

1919
#include <cstdint>
20+
#include <optional>
2021
#include <utility>
2122

2223
#include "src/trace_processor/importers/android_bugreport/android_log_event.h"
2324
#include "src/trace_processor/importers/common/process_tracker.h"
24-
#include "src/trace_processor/tables/android_tables_py.h"
25+
#include "src/trace_processor/storage/trace_storage.h"
26+
#include "src/trace_processor/tables/log_tables_py.h"
2527
#include "src/trace_processor/types/trace_processor_context.h"
2628

2729
namespace perfetto::trace_processor {
2830

2931
AndroidLogEventParser::~AndroidLogEventParser() = default;
3032

3133
void AndroidLogEventParser::Parse(int64_t ts, AndroidLogEvent event) {
32-
tables::AndroidLogTable::Row row;
34+
tables::LogTable::Row row;
3335
row.ts = ts;
34-
row.utid = context_->process_tracker->UpdateThread(event.tid, event.pid);
36+
row.utid = std::make_optional(
37+
context_->process_tracker->UpdateThread(event.tid, event.pid));
3538
row.prio = event.prio;
36-
row.tag = event.tag;
39+
row.log_source = context_->storage->InternString("android");
40+
row.tag = std::make_optional(event.tag);
3741
row.msg = event.msg;
38-
context_->storage->mutable_android_log_table()->Insert(std::move(row));
42+
context_->storage->mutable_log_table()->Insert(std::move(row));
3943
}
4044

4145
} // namespace perfetto::trace_processor

‎src/trace_processor/importers/common/args_tracker.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "src/trace_processor/tables/android_tables_py.h"
3131
#include "src/trace_processor/tables/counter_tables_py.h"
3232
#include "src/trace_processor/tables/flow_tables_py.h"
33+
#include "src/trace_processor/tables/log_tables_py.h"
3334
#include "src/trace_processor/tables/memory_tables_py.h"
3435
#include "src/trace_processor/tables/metadata_tables_py.h"
3536
#include "src/trace_processor/tables/profiler_tables_py.h"
@@ -266,6 +267,10 @@ class ArgsTracker {
266267
return AddArgsTo(context_->storage->mutable_trace_import_logs_table(), id);
267268
}
268269

270+
BoundInserter AddArgsTo(tables::LogTable::Id id) {
271+
return AddArgsTo(context_->storage->mutable_log_table(), id);
272+
}
273+
269274
// Returns a CompactArgSet which contains the args inserted into this
270275
// ArgsTracker. Requires that every arg in this tracker was inserted for the
271276
// "arg_set_id" column given by |column| at the given |row_number|.

‎src/trace_processor/importers/proto/BUILD.gn‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,10 @@ source_set("full") {
166166
"heap_graph_tracker.h",
167167
"jit_tracker.cc",
168168
"jit_tracker.h",
169+
"linux_probes_module.cc",
170+
"linux_probes_module.h",
171+
"linux_probes_parser.cc",
172+
"linux_probes_parser.h",
169173
"metadata_module.cc",
170174
"metadata_module.h",
171175
"pigweed_detokenizer.cc",
@@ -211,6 +215,7 @@ source_set("full") {
211215
"../../../../protos/perfetto/trace/gpu:gpu_interned_data_zero",
212216
"../../../../protos/perfetto/trace/gpu:zero",
213217
"../../../../protos/perfetto/trace/interned_data:zero",
218+
"../../../../protos/perfetto/trace/linux:zero",
214219
"../../../../protos/perfetto/trace/power:zero",
215220
"../../../../protos/perfetto/trace/profiling:zero",
216221
"../../../../protos/perfetto/trace/ps:zero",

‎src/trace_processor/importers/proto/additional_modules.cc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "src/trace_processor/importers/proto/deobfuscation_module.h"
3434
#include "src/trace_processor/importers/proto/graphics_event_module.h"
3535
#include "src/trace_processor/importers/proto/heap_graph_module.h"
36+
#include "src/trace_processor/importers/proto/linux_probes_module.h"
3637
#include "src/trace_processor/importers/proto/metadata_module.h"
3738
#include "src/trace_processor/importers/proto/network_trace_module.h"
3839
#include "src/trace_processor/importers/proto/pixel_modem_module.h"
@@ -62,6 +63,8 @@ void RegisterAdditionalModules(ProtoImporterModuleContext* module_context,
6263
new AndroidKernelWakelocksModule(module_context, context));
6364
module_context->modules.emplace_back(
6465
new AndroidProbesModule(module_context, context));
66+
module_context->modules.emplace_back(
67+
new LinuxProbesModule(module_context, context));
6568
module_context->modules.emplace_back(
6669
new NetworkTraceModule(module_context, context));
6770
module_context->modules.emplace_back(

‎src/trace_processor/importers/proto/android_probes_parser.cc‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
#include "src/trace_processor/storage/metadata.h"
4545
#include "src/trace_processor/storage/stats.h"
4646
#include "src/trace_processor/storage/trace_storage.h"
47+
#include "src/trace_processor/tables/android_tables_py.h"
48+
#include "src/trace_processor/tables/log_tables_py.h"
4749
#include "src/trace_processor/types/trace_processor_context.h"
4850
#include "src/trace_processor/types/variadic.h"
4951

@@ -477,8 +479,14 @@ void AndroidProbesParser::ParseAndroidLogEvent(int64_t ts,
477479

478480
// Log events are NOT required to be sorted by trace_time. The virtual table
479481
// will take care of sorting on-demand.
480-
context_->storage->mutable_android_log_table()->Insert(
481-
{ts, utid, prio, tag_id, msg_id});
482+
tables::LogTable::Row row;
483+
row.ts = ts;
484+
row.utid = tid ? std::make_optional(utid) : std::nullopt;
485+
row.prio = static_cast<uint32_t>(prio);
486+
row.log_source = context_->storage->InternString("android");
487+
row.tag = evt.has_tag() ? std::make_optional(tag_id) : std::nullopt;
488+
row.msg = msg_id;
489+
context_->storage->mutable_log_table()->Insert(row);
482490
}
483491

484492
void AndroidProbesParser::ParseAndroidLogStats(protozero::ConstBytes blob) {

‎src/trace_processor/importers/proto/linux_probes_module.cc‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Copyright (C) 2025 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "src/trace_processor/importers/proto/linux_probes_module.h"
18+
19+
#include <cstdint>
20+
#include <optional>
21+
22+
#include "perfetto/base/logging.h"
23+
#include "perfetto/protozero/field.h"
24+
#include "perfetto/trace_processor/ref_counted.h"
25+
#include "protos/perfetto/common/builtin_clock.pbzero.h"
26+
#include "protos/perfetto/trace/linux/journald_event.pbzero.h"
27+
#include "protos/perfetto/trace/trace_packet.pbzero.h"
28+
#include "src/trace_processor/importers/common/clock_tracker.h"
29+
#include "src/trace_processor/importers/common/parser_types.h"
30+
#include "src/trace_processor/importers/proto/blob_packet_writer.h"
31+
#include "src/trace_processor/importers/proto/packet_sequence_state_generation.h"
32+
#include "src/trace_processor/importers/proto/proto_importer_module.h"
33+
#include "src/trace_processor/sorter/trace_sorter.h"
34+
#include "src/trace_processor/types/trace_processor_context.h"
35+
36+
namespace perfetto::trace_processor {
37+
38+
using perfetto::protos::pbzero::TracePacket;
39+
40+
LinuxProbesModule::LinuxProbesModule(ProtoImporterModuleContext* module_context,
41+
TraceProcessorContext* context)
42+
: ProtoImporterModule(module_context), parser_(context), context_(context) {
43+
RegisterForField(TracePacket::kJournaldEventFieldNumber);
44+
}
45+
46+
ModuleResult LinuxProbesModule::TokenizePacket(
47+
const protos::pbzero::TracePacket_Decoder&,
48+
TraceBlobView* packet,
49+
int64_t packet_timestamp,
50+
RefPtr<PacketSequenceStateGeneration> state,
51+
uint32_t field_id) {
52+
if (field_id != TracePacket::kJournaldEventFieldNumber) {
53+
return ModuleResult::Ignored();
54+
}
55+
56+
protos::pbzero::TracePacket::Decoder decoder(packet->data(),
57+
packet->length());
58+
auto journald_bytes = decoder.journald_event();
59+
protos::pbzero::JournaldEventPacket::Decoder pkt(journald_bytes);
60+
protos::pbzero::JournaldEventPacket::JournaldEvent::Decoder evt(pkt.event());
61+
62+
TraceBlobView tbv =
63+
context_->blob_packet_writer->WritePacket([&](auto* data_packet) {
64+
data_packet->set_timestamp(static_cast<uint64_t>(packet_timestamp));
65+
auto* jpkt = data_packet->set_journald_event();
66+
auto* jevt = jpkt->set_event();
67+
if (evt.has_pid())
68+
jevt->set_pid(evt.pid());
69+
if (evt.has_tid())
70+
jevt->set_tid(evt.tid());
71+
if (evt.has_uid())
72+
jevt->set_uid(evt.uid());
73+
if (evt.has_gid())
74+
jevt->set_gid(evt.gid());
75+
if (evt.has_prio())
76+
jevt->set_prio(evt.prio());
77+
if (evt.has_tag())
78+
jevt->set_tag(evt.tag());
79+
if (evt.has_message())
80+
jevt->set_message(evt.message());
81+
if (evt.has_comm())
82+
jevt->set_comm(evt.comm());
83+
if (evt.has_exe())
84+
jevt->set_exe(evt.exe());
85+
if (evt.has_systemd_unit())
86+
jevt->set_systemd_unit(evt.systemd_unit());
87+
if (evt.has_hostname())
88+
jevt->set_hostname(evt.hostname());
89+
if (evt.has_transport())
90+
jevt->set_transport(evt.transport());
91+
});
92+
module_context_->trace_packet_stream->Push(
93+
packet_timestamp, TracePacketData{std::move(tbv), state});
94+
return ModuleResult::Handled();
95+
}
96+
97+
void LinuxProbesModule::ParseTracePacketData(
98+
const protos::pbzero::TracePacket_Decoder& decoder,
99+
int64_t ts,
100+
const TracePacketData&,
101+
uint32_t field_id) {
102+
switch (field_id) {
103+
case TracePacket::kJournaldEventFieldNumber:
104+
parser_.ParseJournaldPacket(ts, decoder.journald_event());
105+
return;
106+
default:
107+
PERFETTO_FATAL("Unexpected field_id in LinuxProbesModule");
108+
}
109+
}
110+
111+
} // namespace perfetto::trace_processor

‎src/trace_processor/importers/proto/linux_probes_module.h‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright (C) 2025 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_LINUX_PROBES_MODULE_H_
18+
#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_LINUX_PROBES_MODULE_H_
19+
20+
#include "src/trace_processor/importers/proto/linux_probes_parser.h"
21+
#include "src/trace_processor/importers/proto/proto_importer_module.h"
22+
23+
namespace perfetto::trace_processor {
24+
25+
class TraceProcessorContext;
26+
class LinuxProbesModule : public ProtoImporterModule {
27+
public:
28+
explicit LinuxProbesModule(ProtoImporterModuleContext* module_context,
29+
TraceProcessorContext* context);
30+
31+
ModuleResult TokenizePacket(const protos::pbzero::TracePacket_Decoder&,
32+
TraceBlobView* packet,
33+
int64_t packet_timestamp,
34+
RefPtr<PacketSequenceStateGeneration> state,
35+
uint32_t field_id) override;
36+
37+
void ParseTracePacketData(const protos::pbzero::TracePacket_Decoder& decoder,
38+
int64_t ts,
39+
const TracePacketData&,
40+
uint32_t field_id) override;
41+
42+
private:
43+
LinuxProbesParser parser_;
44+
TraceProcessorContext* context_ = nullptr;
45+
};
46+
47+
} // namespace perfetto::trace_processor
48+
49+
#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_LINUX_PROBES_MODULE_H_

‎src/trace_processor/importers/proto/linux_probes_parser.cc‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
/*
2+
* Copyright (C) 2025 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "src/trace_processor/importers/proto/linux_probes_parser.h"
18+
19+
#include <cstdint>
20+
#include <optional>
21+
22+
#include "perfetto/ext/base/string_view.h"
23+
#include "protos/perfetto/trace/linux/journald_event.pbzero.h"
24+
#include "src/trace_processor/importers/common/args_tracker.h"
25+
#include "src/trace_processor/importers/common/process_tracker.h"
26+
#include "src/trace_processor/storage/trace_storage.h"
27+
#include "src/trace_processor/tables/log_tables_py.h"
28+
#include "src/trace_processor/types/trace_processor_context.h"
29+
#include "src/trace_processor/types/variadic.h"
30+
31+
namespace perfetto::trace_processor {
32+
33+
LinuxProbesParser::LinuxProbesParser(TraceProcessorContext* context)
34+
: context_(context),
35+
uid_key_id_(context->storage->InternString("uid")),
36+
comm_key_id_(context->storage->InternString("comm")),
37+
systemd_unit_key_id_(context->storage->InternString("systemd_unit")),
38+
hostname_key_id_(context->storage->InternString("hostname")),
39+
transport_key_id_(context->storage->InternString("transport")),
40+
journald_source_id_(context->storage->InternString("journald")) {}
41+
42+
void LinuxProbesParser::ParseJournaldPacket(int64_t ts,
43+
protozero::ConstBytes blob) {
44+
protos::pbzero::JournaldEventPacket::Decoder pkt(blob);
45+
ParseJournaldEvent(ts, pkt.event());
46+
}
47+
48+
void LinuxProbesParser::ParseJournaldEvent(int64_t ts,
49+
protozero::ConstBytes blob) {
50+
protos::pbzero::JournaldEventPacket::JournaldEvent::Decoder evt(blob);
51+
52+
auto pid = evt.has_pid() ? static_cast<uint32_t>(evt.pid()) : 0u;
53+
auto tid = evt.has_tid() ? static_cast<uint32_t>(evt.tid()) : pid;
54+
55+
std::optional<uint32_t> utid;
56+
if (pid != 0) {
57+
utid = context_->process_tracker->UpdateThread(tid, pid);
58+
}
59+
60+
auto prio = static_cast<uint32_t>(evt.has_prio() ? evt.prio() : 0u);
61+
62+
StringId tag_id = evt.has_tag() ? context_->storage->InternString(evt.tag())
63+
: kNullStringId;
64+
StringId msg_id = context_->storage->InternString(
65+
evt.has_message() ? evt.message() : base::StringView());
66+
67+
std::optional<uint32_t> uid;
68+
if (evt.has_uid())
69+
uid = static_cast<uint32_t>(evt.uid());
70+
71+
StringId comm_id = evt.has_comm()
72+
? context_->storage->InternString(evt.comm())
73+
: kNullStringId;
74+
StringId unit_id = evt.has_systemd_unit()
75+
? context_->storage->InternString(evt.systemd_unit())
76+
: kNullStringId;
77+
StringId host_id = evt.has_hostname()
78+
? context_->storage->InternString(evt.hostname())
79+
: kNullStringId;
80+
StringId transport_id = evt.has_transport()
81+
? context_->storage->InternString(evt.transport())
82+
: kNullStringId;
83+
84+
tables::LogTable::Row row;
85+
row.ts = ts;
86+
row.utid = utid;
87+
row.prio = prio;
88+
row.log_source = journald_source_id_;
89+
row.tag = evt.has_tag() ? std::make_optional(tag_id) : std::nullopt;
90+
row.msg = msg_id;
91+
auto id = context_->storage->mutable_log_table()->Insert(row).id;
92+
93+
if (uid || evt.has_comm() || evt.has_systemd_unit() || evt.has_hostname() ||
94+
evt.has_transport()) {
95+
ArgsTracker args_tracker(context_);
96+
auto inserter = args_tracker.AddArgsTo(id);
97+
if (uid) {
98+
inserter.AddArg(uid_key_id_, uid_key_id_,
99+
Variadic::Integer(static_cast<int64_t>(*uid)));
100+
}
101+
if (evt.has_comm()) {
102+
inserter.AddArg(comm_key_id_, comm_key_id_, Variadic::String(comm_id));
103+
}
104+
if (evt.has_systemd_unit()) {
105+
inserter.AddArg(systemd_unit_key_id_, systemd_unit_key_id_,
106+
Variadic::String(unit_id));
107+
}
108+
if (evt.has_hostname()) {
109+
inserter.AddArg(hostname_key_id_, hostname_key_id_,
110+
Variadic::String(host_id));
111+
}
112+
if (evt.has_transport()) {
113+
inserter.AddArg(transport_key_id_, transport_key_id_,
114+
Variadic::String(transport_id));
115+
}
116+
}
117+
}
118+
119+
} // namespace perfetto::trace_processor

0 commit comments

Comments
 (0)