Skip to content

Commit a7787d8

Browse files
staticfloatstaticfloat
committed
tp: Parse JournaldEventPacket into unified LogTable
Refactor `AndroidLogTable` into a unified `LogTable` (`log_tables.py`, exposed as `__intrinsic_logs`) that stores log entries from all sources. A new log_source STRING column identifies the origin: 'android' or 'journald'. `LogTable` is moved out of `android_tables.py` into its own `log_tables.py` since it is no longer Android-specific. New linux_probes_parser.cc parses `JournaldEventPacket` and inserts into `LogTable` with log_source='journald'. Journald-specific metadata (uid, comm, systemd_unit, hostname, transport) is stored via `ArgsTracker`. Three SQL views are provided in the prelude: - logs: all entries from all sources - android_logs: convenience alias for log_source='android' entries - journald_logs: journald entries with metadata columns via extract_arg() Diff tests updated and a new Linux diff test suite covers journald parsing.
1 parent c13bac7 commit a7787d8

26 files changed

Lines changed: 584 additions & 62 deletions

‎Android.bp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16559,6 +16559,8 @@ filegroup {
1655916559
"src/trace_processor/importers/proto/heap_graph_module.cc",
1656016560
"src/trace_processor/importers/proto/heap_graph_tracker.cc",
1656116561
"src/trace_processor/importers/proto/jit_tracker.cc",
16562+
"src/trace_processor/importers/proto/linux_probes_module.cc",
16563+
"src/trace_processor/importers/proto/linux_probes_parser.cc",
1656216564
"src/trace_processor/importers/proto/metadata_module.cc",
1656316565
"src/trace_processor/importers/proto/pigweed_detokenizer.cc",
1656416566
"src/trace_processor/importers/proto/pixel_modem_module.cc",
@@ -17402,6 +17404,7 @@ filegroup {
1740217404
"src/trace_processor/perfetto_sql/stdlib/linux/cpu/utilization/thread_cpu.sql",
1740317405
"src/trace_processor/perfetto_sql/stdlib/linux/devfreq.sql",
1740417406
"src/trace_processor/perfetto_sql/stdlib/linux/irqs.sql",
17407+
"src/trace_processor/perfetto_sql/stdlib/linux/journald.sql",
1740517408
"src/trace_processor/perfetto_sql/stdlib/linux/memory/general.sql",
1740617409
"src/trace_processor/perfetto_sql/stdlib/linux/memory/high_watermark.sql",
1740717410
"src/trace_processor/perfetto_sql/stdlib/linux/memory/process.sql",
@@ -17713,6 +17716,7 @@ python_binary_host {
1771317716
"src/trace_processor/tables/etm_tables.py",
1771417717
"src/trace_processor/tables/flow_tables.py",
1771517718
"src/trace_processor/tables/jit_tables.py",
17719+
"src/trace_processor/tables/log_tables.py",
1771617720
"src/trace_processor/tables/memory_tables.py",
1771717721
"src/trace_processor/tables/metadata_tables.py",
1771817722
"src/trace_processor/tables/perf_tables.py",
@@ -17798,6 +17802,7 @@ python_binary_host {
1779817802
"src/trace_processor/tables/etm_tables.py",
1779917803
"src/trace_processor/tables/flow_tables.py",
1780017804
"src/trace_processor/tables/jit_tables.py",
17805+
"src/trace_processor/tables/log_tables.py",
1780117806
"src/trace_processor/tables/memory_tables.py",
1780217807
"src/trace_processor/tables/metadata_tables.py",
1780317808
"src/trace_processor/tables/perf_tables.py",
@@ -18091,6 +18096,7 @@ python_binary_host {
1809118096
"src/trace_processor/tables/etm_tables.py",
1809218097
"src/trace_processor/tables/flow_tables.py",
1809318098
"src/trace_processor/tables/jit_tables.py",
18099+
"src/trace_processor/tables/log_tables.py",
1809418100
"src/trace_processor/tables/memory_tables.py",
1809518101
"src/trace_processor/tables/metadata_tables.py",
1809618102
"src/trace_processor/tables/perf_tables.py",
@@ -18153,6 +18159,7 @@ python_binary_host {
1815318159
"src/trace_processor/tables/etm_tables.py",
1815418160
"src/trace_processor/tables/flow_tables.py",
1815518161
"src/trace_processor/tables/jit_tables.py",
18162+
"src/trace_processor/tables/log_tables.py",
1815618163
"src/trace_processor/tables/memory_tables.py",
1815718164
"src/trace_processor/tables/metadata_tables.py",
1815818165
"src/trace_processor/tables/perf_tables.py",
@@ -18214,6 +18221,7 @@ python_binary_host {
1821418221
"src/trace_processor/tables/etm_tables.py",
1821518222
"src/trace_processor/tables/flow_tables.py",
1821618223
"src/trace_processor/tables/jit_tables.py",
18224+
"src/trace_processor/tables/log_tables.py",
1821718225
"src/trace_processor/tables/memory_tables.py",
1821818226
"src/trace_processor/tables/metadata_tables.py",
1821918227
"src/trace_processor/tables/perf_tables.py",
@@ -18967,6 +18975,7 @@ genrule {
1896718975
"src/trace_processor/tables/etm_tables.py",
1896818976
"src/trace_processor/tables/flow_tables.py",
1896918977
"src/trace_processor/tables/jit_tables.py",
18978+
"src/trace_processor/tables/log_tables.py",
1897018979
"src/trace_processor/tables/memory_tables.py",
1897118980
"src/trace_processor/tables/metadata_tables.py",
1897218981
"src/trace_processor/tables/perf_tables.py",
@@ -18994,6 +19003,8 @@ genrule {
1899419003
"src/trace_processor/tables/flow_tables_py.h",
1899519004
"src/trace_processor/tables/jit_tables_fwd.h",
1899619005
"src/trace_processor/tables/jit_tables_py.h",
19006+
"src/trace_processor/tables/log_tables_fwd.h",
19007+
"src/trace_processor/tables/log_tables_py.h",
1899719008
"src/trace_processor/tables/memory_tables_fwd.h",
1899819009
"src/trace_processor/tables/memory_tables_py.h",
1899919010
"src/trace_processor/tables/metadata_tables_fwd.h",
@@ -19029,6 +19040,7 @@ python_binary_host {
1902919040
"src/trace_processor/tables/etm_tables.py",
1903019041
"src/trace_processor/tables/flow_tables.py",
1903119042
"src/trace_processor/tables/jit_tables.py",
19043+
"src/trace_processor/tables/log_tables.py",
1903219044
"src/trace_processor/tables/memory_tables.py",
1903319045
"src/trace_processor/tables/metadata_tables.py",
1903419046
"src/trace_processor/tables/perf_tables.py",

‎BUILD‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2973,6 +2973,10 @@ perfetto_filegroup(
29732973
"src/trace_processor/importers/proto/heap_graph_tracker.h",
29742974
"src/trace_processor/importers/proto/jit_tracker.cc",
29752975
"src/trace_processor/importers/proto/jit_tracker.h",
2976+
"src/trace_processor/importers/proto/linux_probes_module.cc",
2977+
"src/trace_processor/importers/proto/linux_probes_module.h",
2978+
"src/trace_processor/importers/proto/linux_probes_parser.cc",
2979+
"src/trace_processor/importers/proto/linux_probes_parser.h",
29762980
"src/trace_processor/importers/proto/metadata_module.cc",
29772981
"src/trace_processor/importers/proto/metadata_module.h",
29782982
"src/trace_processor/importers/proto/pigweed_detokenizer.cc",
@@ -3991,6 +3995,7 @@ perfetto_filegroup(
39913995
"src/trace_processor/perfetto_sql/stdlib/linux/block_io.sql",
39923996
"src/trace_processor/perfetto_sql/stdlib/linux/devfreq.sql",
39933997
"src/trace_processor/perfetto_sql/stdlib/linux/irqs.sql",
3998+
"src/trace_processor/perfetto_sql/stdlib/linux/journald.sql",
39943999
"src/trace_processor/perfetto_sql/stdlib/linux/threads.sql",
39954000
],
39964001
)
@@ -5308,6 +5313,7 @@ perfetto_cc_tp_tables(
53085313
"src/trace_processor/tables/etm_tables.py",
53095314
"src/trace_processor/tables/flow_tables.py",
53105315
"src/trace_processor/tables/jit_tables.py",
5316+
"src/trace_processor/tables/log_tables.py",
53115317
"src/trace_processor/tables/memory_tables.py",
53125318
"src/trace_processor/tables/metadata_tables.py",
53135319
"src/trace_processor/tables/perf_tables.py",
@@ -5331,6 +5337,8 @@ perfetto_cc_tp_tables(
53315337
"src/trace_processor/tables/flow_tables_py.h",
53325338
"src/trace_processor/tables/jit_tables_fwd.h",
53335339
"src/trace_processor/tables/jit_tables_py.h",
5340+
"src/trace_processor/tables/log_tables_fwd.h",
5341+
"src/trace_processor/tables/log_tables_py.h",
53345342
"src/trace_processor/tables/memory_tables_fwd.h",
53355343
"src/trace_processor/tables/memory_tables_py.h",
53365344
"src/trace_processor/tables/metadata_tables_fwd.h",

‎src/trace_processor/importers/android_bugreport/android_log_event_parser.cc‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,29 @@
1717
#include "src/trace_processor/importers/android_bugreport/android_log_event_parser.h"
1818

1919
#include <cstdint>
20+
#include <optional>
2021
#include <utility>
2122

2223
#include "src/trace_processor/importers/android_bugreport/android_log_event.h"
2324
#include "src/trace_processor/importers/common/process_tracker.h"
24-
#include "src/trace_processor/tables/android_tables_py.h"
25+
#include "src/trace_processor/storage/trace_storage.h"
26+
#include "src/trace_processor/tables/log_tables_py.h"
2527
#include "src/trace_processor/types/trace_processor_context.h"
2628

2729
namespace perfetto::trace_processor {
2830

2931
AndroidLogEventParser::~AndroidLogEventParser() = default;
3032

3133
void AndroidLogEventParser::Parse(int64_t ts, AndroidLogEvent event) {
32-
tables::AndroidLogTable::Row row;
34+
tables::LogTable::Row row;
3335
row.ts = ts;
34-
row.utid = context_->process_tracker->UpdateThread(event.tid, event.pid);
36+
row.utid = std::make_optional(
37+
context_->process_tracker->UpdateThread(event.tid, event.pid));
3538
row.prio = event.prio;
36-
row.tag = event.tag;
39+
row.log_source = context_->storage->InternString("android_logcat");
40+
row.tag = std::make_optional(event.tag);
3741
row.msg = event.msg;
38-
context_->storage->mutable_android_log_table()->Insert(std::move(row));
42+
context_->storage->mutable_log_table()->Insert(std::move(row));
3943
}
4044

4145
} // namespace perfetto::trace_processor

‎src/trace_processor/importers/common/args_tracker.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "src/trace_processor/tables/android_tables_py.h"
3131
#include "src/trace_processor/tables/counter_tables_py.h"
3232
#include "src/trace_processor/tables/flow_tables_py.h"
33+
#include "src/trace_processor/tables/log_tables_py.h"
3334
#include "src/trace_processor/tables/memory_tables_py.h"
3435
#include "src/trace_processor/tables/metadata_tables_py.h"
3536
#include "src/trace_processor/tables/profiler_tables_py.h"
@@ -266,6 +267,10 @@ class ArgsTracker {
266267
return AddArgsTo(context_->storage->mutable_trace_import_logs_table(), id);
267268
}
268269

270+
BoundInserter AddArgsTo(tables::LogTable::Id id) {
271+
return AddArgsTo(context_->storage->mutable_log_table(), id);
272+
}
273+
269274
// Returns a CompactArgSet which contains the args inserted into this
270275
// ArgsTracker. Requires that every arg in this tracker was inserted for the
271276
// "arg_set_id" column given by |column| at the given |row_number|.

‎src/trace_processor/importers/proto/BUILD.gn‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,10 @@ source_set("full") {
166166
"heap_graph_tracker.h",
167167
"jit_tracker.cc",
168168
"jit_tracker.h",
169+
"linux_probes_module.cc",
170+
"linux_probes_module.h",
171+
"linux_probes_parser.cc",
172+
"linux_probes_parser.h",
169173
"metadata_module.cc",
170174
"metadata_module.h",
171175
"pigweed_detokenizer.cc",
@@ -211,6 +215,7 @@ source_set("full") {
211215
"../../../../protos/perfetto/trace/gpu:gpu_interned_data_zero",
212216
"../../../../protos/perfetto/trace/gpu:zero",
213217
"../../../../protos/perfetto/trace/interned_data:zero",
218+
"../../../../protos/perfetto/trace/linux:zero",
214219
"../../../../protos/perfetto/trace/power:zero",
215220
"../../../../protos/perfetto/trace/profiling:zero",
216221
"../../../../protos/perfetto/trace/ps:zero",

‎src/trace_processor/importers/proto/additional_modules.cc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "src/trace_processor/importers/proto/deobfuscation_module.h"
3333
#include "src/trace_processor/importers/proto/graphics_event_module.h"
3434
#include "src/trace_processor/importers/proto/heap_graph_module.h"
35+
#include "src/trace_processor/importers/proto/linux_probes_module.h"
3536
#include "src/trace_processor/importers/proto/metadata_module.h"
3637
#include "src/trace_processor/importers/proto/network_trace_module.h"
3738
#include "src/trace_processor/importers/proto/pixel_modem_module.h"
@@ -57,6 +58,8 @@ void RegisterAdditionalModules(ProtoImporterModuleContext* module_context,
5758
new AndroidKernelWakelocksModule(module_context, context));
5859
module_context->modules.emplace_back(
5960
new AndroidProbesModule(module_context, context));
61+
module_context->modules.emplace_back(
62+
new LinuxProbesModule(module_context, context));
6063
module_context->modules.emplace_back(
6164
new NetworkTraceModule(module_context, context));
6265
module_context->modules.emplace_back(

‎src/trace_processor/importers/proto/android_probes_parser.cc‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444
#include "src/trace_processor/storage/metadata.h"
4545
#include "src/trace_processor/storage/stats.h"
4646
#include "src/trace_processor/storage/trace_storage.h"
47+
#include "src/trace_processor/tables/android_tables_py.h"
48+
#include "src/trace_processor/tables/log_tables_py.h"
4749
#include "src/trace_processor/types/trace_processor_context.h"
4850
#include "src/trace_processor/types/variadic.h"
4951

@@ -477,8 +479,14 @@ void AndroidProbesParser::ParseAndroidLogEvent(int64_t ts,
477479

478480
// Log events are NOT required to be sorted by trace_time. The virtual table
479481
// will take care of sorting on-demand.
480-
context_->storage->mutable_android_log_table()->Insert(
481-
{ts, utid, prio, tag_id, msg_id});
482+
tables::LogTable::Row row;
483+
row.ts = ts;
484+
row.utid = tid ? std::make_optional(utid) : std::nullopt;
485+
row.prio = static_cast<uint32_t>(prio);
486+
row.log_source = context_->storage->InternString("android_logcat");
487+
row.tag = evt.has_tag() ? std::make_optional(tag_id) : std::nullopt;
488+
row.msg = msg_id;
489+
context_->storage->mutable_log_table()->Insert(row);
482490
}
483491

484492
void AndroidProbesParser::ParseAndroidLogStats(protozero::ConstBytes blob) {

‎src/trace_processor/importers/proto/linux_probes_module.cc‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright (C) 2025 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "src/trace_processor/importers/proto/linux_probes_module.h"
18+
19+
#include <cstdint>
20+
#include <optional>
21+
22+
#include "perfetto/base/logging.h"
23+
#include "perfetto/protozero/field.h"
24+
#include "perfetto/trace_processor/ref_counted.h"
25+
#include "protos/perfetto/common/builtin_clock.pbzero.h"
26+
#include "protos/perfetto/trace/linux/journald_event.pbzero.h"
27+
#include "protos/perfetto/trace/trace_packet.pbzero.h"
28+
#include "src/trace_processor/importers/common/clock_tracker.h"
29+
#include "src/trace_processor/importers/common/parser_types.h"
30+
#include "src/trace_processor/importers/proto/blob_packet_writer.h"
31+
#include "src/trace_processor/importers/proto/packet_sequence_state_generation.h"
32+
#include "src/trace_processor/importers/proto/proto_importer_module.h"
33+
#include "src/trace_processor/sorter/trace_sorter.h"
34+
#include "src/trace_processor/types/trace_processor_context.h"
35+
36+
namespace perfetto::trace_processor {
37+
38+
using perfetto::protos::pbzero::TracePacket;
39+
40+
LinuxProbesModule::LinuxProbesModule(ProtoImporterModuleContext* module_context,
41+
TraceProcessorContext* context)
42+
: ProtoImporterModule(module_context), parser_(context) {
43+
RegisterForField(TracePacket::kJournaldEventFieldNumber);
44+
}
45+
46+
void LinuxProbesModule::ParseTracePacketData(
47+
const protos::pbzero::TracePacket_Decoder& decoder,
48+
int64_t ts,
49+
const TracePacketData&,
50+
uint32_t field_id) {
51+
switch (field_id) {
52+
case TracePacket::kJournaldEventFieldNumber:
53+
parser_.ParseSystemdJournaldEvent(ts, decoder.journald_event());
54+
return;
55+
default:
56+
PERFETTO_FATAL("Unexpected field_id in LinuxProbesModule");
57+
}
58+
}
59+
60+
} // namespace perfetto::trace_processor

‎src/trace_processor/importers/proto/linux_probes_module.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright (C) 2025 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_LINUX_PROBES_MODULE_H_
18+
#define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_LINUX_PROBES_MODULE_H_
19+
20+
#include "src/trace_processor/importers/proto/linux_probes_parser.h"
21+
#include "src/trace_processor/importers/proto/proto_importer_module.h"
22+
23+
namespace perfetto::trace_processor {
24+
25+
class TraceProcessorContext;
26+
class LinuxProbesModule : public ProtoImporterModule {
27+
public:
28+
explicit LinuxProbesModule(ProtoImporterModuleContext* module_context,
29+
TraceProcessorContext* context);
30+
31+
void ParseTracePacketData(const protos::pbzero::TracePacket_Decoder& decoder,
32+
int64_t ts,
33+
const TracePacketData&,
34+
uint32_t field_id) override;
35+
36+
private:
37+
LinuxProbesParser parser_;
38+
};
39+
40+
} // namespace perfetto::trace_processor
41+
42+
#endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_LINUX_PROBES_MODULE_H_

0 commit comments

Comments
 (0)