Skip to content

Commit bec742d

Browse files
KayzzzZKayzzzZ
committed
1. push span/metrics自监控指标下移到NetworkObserverManager中
2. timeDiff替换成 KernelTimeNanoToUTC方法 3. 移除Periodical Event，在Timer Event Execute方法中主动PushNextEvent 4. eBPF Driver 中定义错误码 5. 采集配置中移除测试相关的配置 6. span和stringview 目录结构移动到 common 目录下
1 parent cbe16e6 commit bec742d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+503
-480
lines changed

Diff for: ‎core/CMakeLists.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ set(SUB_DIRECTORIES_LIST
126126
protobuf/sls protobuf/models
127127
file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling
128128
prometheus prometheus/labels prometheus/schedulers prometheus/async prometheus/component
129-
ebpf ebpf/type/table ebpf/util ebpf/util/sampler ebpf/protocol/http ebpf/protocol ebpf/plugin/file_security ebpf/plugin/network_observer ebpf/plugin/process_security ebpf/plugin/network_security ebpf/plugin ebpf/observer ebpf/security
129+
ebpf ebpf/type ebpf/type/table ebpf/util ebpf/util/sampler ebpf/protocol/http ebpf/protocol ebpf/plugin/file_security ebpf/plugin/network_observer ebpf/plugin/process_security ebpf/plugin/network_security ebpf/plugin ebpf/observer ebpf/security
130130
parser
131131
host_monitor host_monitor/collector
132132
)

Diff for: ‎core/collection_pipeline/batch/BatchItem.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
#include "collection_pipeline/batch/BatchStatus.h"
2424
#include "collection_pipeline/batch/BatchedEvents.h"
2525
#include "collection_pipeline/batch/FlushStrategy.h"
26+
#include "common/StringView.h"
2627
#include "models/PipelineEventGroup.h"
27-
#include "models/StringView.h"
2828

2929
namespace logtail {
3030

Diff for: ‎core/collection_pipeline/batch/BatchedEvents.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919
#include <unordered_set>
2020
#include <vector>
2121

22+
#include "common/StringView.h"
2223
#include "models/PipelineEventGroup.h"
23-
#include "models/StringView.h"
2424

2525
namespace logtail {
2626

Diff for: ‎core/common/MachineInfoUtil.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
#include "json/value.h"
2424

2525
#include "AppConfig.h"
26-
#include "models/StringView.h"
26+
#include "common/StringView.h"
2727

2828
namespace logtail {
2929

Diff for: ‎core/models/Span.h renamed to ‎core/common/Span.h

File renamed without changes.

Diff for: ‎core/common/StringTools.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
#include <string>
2828
#include <vector>
2929

30-
#include "models/StringView.h"
30+
#include "common/StringView.h"
3131

3232
namespace logtail {
3333

Diff for: ‎core/models/StringView.h renamed to ‎core/common/StringView.h

File renamed without changes.

Diff for: ‎core/common/TimeUtil.cpp

+4-6
Original file line numberDiff line numberDiff line change
@@ -428,14 +428,12 @@ std::chrono::nanoseconds GetTimeDiffFromMonotonic() {
428428
#endif
429429
}
430430

431-
struct timespec KernelNanoTimeToUTC(uint64_t nano) {
432-
static std::chrono::nanoseconds diff = GetTimeDiffFromMonotonic();
433-
auto ts = std::chrono::nanoseconds(nano + diff.count());
431+
struct timespec KernelTimeNanoToUTC(uint64_t nano) {
432+
static auto diff = GetTimeDiffFromMonotonic().count();
433+
auto ts = std::chrono::nanoseconds(nano + diff);
434434
auto seconds = std::chrono::duration_cast<std::chrono::seconds>(ts);
435435
auto nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(ts - seconds);
436-
struct timespec res;
437-
res.tv_sec = seconds.count();
438-
res.tv_nsec = nanoseconds.count();
436+
struct timespec res = {seconds.count(), nanoseconds.count()};
439437
return res;
440438
}
441439

Diff for: ‎core/common/TimeUtil.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,6 @@ long GetTicksPerSecond();
101101

102102
std::chrono::nanoseconds GetTimeDiffFromMonotonic();
103103

104-
struct timespec KernelNanoTimeToUTC(uint64_t nano);
104+
struct timespec KernelTimeNanoToUTC(uint64_t nano);
105105

106106
} // namespace logtail

Diff for: ‎core/common/memory/SourceBuffer.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
#include <list>
2020
#include <memory>
2121

22-
#include "models/StringView.h"
22+
#include "common/StringView.h"
2323

2424
namespace logtail {
2525

Diff for: ‎core/common/timer/Timer.cpp

+9-9
Original file line numberDiff line numberDiff line change
@@ -90,15 +90,15 @@ void Timer::Run() {
9090
LOG_INFO(sLogger, ("invalid timer event", "task is cancelled"));
9191
} else {
9292
e->Execute();
93-
if (e->IsPeriodicalEvent()) {
94-
auto pe = static_cast<PeriodicalTimerEvent*>(e.get());
95-
pe->ScheduleNext();
96-
if (!pe->IsStop()) {
97-
PushEvent(std::move(e));
98-
} else {
99-
LOG_DEBUG(sLogger, ("periodical event schedule done", "exit"));
100-
}
101-
}
93+
// if (e->IsPeriodicalEvent()) {
94+
// auto pe = static_cast<PeriodicalTimerEvent*>(e.get());
95+
// pe->ScheduleNext();
96+
// if (!pe->IsStop()) {
97+
// PushEvent(std::move(e));
98+
// } else {
99+
// LOG_DEBUG(sLogger, ("periodical event schedule done", "exit"));
100+
// }
101+
// }
102102
}
103103
queueLock.lock();
104104
}

Diff for: ‎core/common/timer/TimerEvent.h

+16-15
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ class TimerEvent {
2727

2828
virtual bool IsValid() const = 0;
2929
virtual bool Execute() = 0;
30-
virtual bool IsPeriodicalEvent() { return false; }
30+
// virtual bool IsPeriodicalEvent() { return false; }
3131

3232
std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; }
3333
void SetExecTime(std::chrono::steady_clock::time_point nextExecTime) { mExecTime = nextExecTime; }
@@ -36,22 +36,23 @@ class TimerEvent {
3636
std::chrono::steady_clock::time_point mExecTime;
3737
};
3838

39-
class PeriodicalTimerEvent : public TimerEvent {
40-
public:
41-
PeriodicalTimerEvent(int intervalSec)
42-
: TimerEvent(std::chrono::steady_clock::now() + std::chrono::seconds(intervalSec)), mIntervalSec(intervalSec) {}
43-
virtual ~PeriodicalTimerEvent() = default;
39+
// class PeriodicalTimerEvent : public TimerEvent {
40+
// public:
41+
// PeriodicalTimerEvent(int intervalSec)
42+
// : TimerEvent(std::chrono::steady_clock::now() + std::chrono::seconds(intervalSec)), mIntervalSec(intervalSec)
43+
// {}
44+
// virtual ~PeriodicalTimerEvent() = default;
4445

45-
virtual bool IsValid() const = 0;
46-
virtual bool Execute() = 0;
47-
virtual bool IsPeriodicalEvent() override { return true; }
48-
virtual void ScheduleNext() { mExecTime += std::chrono::seconds(mIntervalSec); }
49-
virtual bool IsStop() = 0;
46+
// virtual bool IsValid() const = 0;
47+
// virtual bool Execute() = 0;
48+
// virtual bool IsPeriodicalEvent() override { return true; }
49+
// virtual void ScheduleNext() { mExecTime += std::chrono::seconds(mIntervalSec); }
50+
// virtual bool IsStop() = 0;
5051

51-
std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; }
52+
// std::chrono::steady_clock::time_point GetExecTime() const { return mExecTime; }
5253

53-
protected:
54-
int mIntervalSec;
55-
};
54+
// protected:
55+
// int mIntervalSec;
56+
// };
5657

5758
} // namespace logtail

Diff for: ‎core/ebpf/Config.cpp

-11
Original file line numberDiff line numberDiff line change
@@ -84,17 +84,6 @@ bool InitObserverNetworkOptionInner(const Json::Value& probeConfig,
8484
mContext->GetLogstoreName(),
8585
mContext->GetRegion());
8686
}
87-
// DisableMetadata (Optional)
88-
if (!GetOptionalBoolParam(probeConfig, "DisableMetadata", thisObserverNetworkOption.mDisableMetadata, errorMsg)) {
89-
PARAM_WARNING_IGNORE(mContext->GetLogger(),
90-
mContext->GetAlarm(),
91-
errorMsg,
92-
sName,
93-
mContext->GetConfigName(),
94-
mContext->GetProjectName(),
95-
mContext->GetLogstoreName(),
96-
mContext->GetRegion());
97-
}
9887
// SampleRate (Optional)
9988
if (!GetOptionalDoubleParam(probeConfig, "SampleRate", thisObserverNetworkOption.mSampleRate, errorMsg)) {
10089
PARAM_WARNING_IGNORE(mContext->GetLogger(),

Diff for: ‎core/ebpf/driver/CallName.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
namespace logtail {
2222
namespace ebpf {
2323

24+
#define ERR_UNKNOWN_CALLNAME -1
25+
2426
static inline int GetCallNameIdx(const std::string& call_name) {
2527
if (call_name == "security_file_permission") {
2628
return SECURE_FUNC_TRACEPOINT_FUNC_SECURITY_FILE_PERMISSION;
@@ -41,7 +43,7 @@ static inline int GetCallNameIdx(const std::string& call_name) {
4143
}
4244
ebpf_log(
4345
logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN, "[GetCallNameIdx] unknown call name: %s \n", call_name.c_str());
44-
return -1;
46+
return ERR_UNKNOWN_CALLNAME;
4547
}
4648

4749
} // namespace ebpf

Diff for: ‎core/ebpf/driver/FileFilter.cpp

+46-44
Original file line numberDiff line numberDiff line change
@@ -38,49 +38,50 @@ namespace logtail {
3838
namespace ebpf {
3939

4040
int CreateFileFilterForCallname(std::shared_ptr<logtail::ebpf::BPFWrapper<security_bpf>> wrapper,
41-
const std::string& call_name,
41+
const std::string& callName,
4242
const std::variant<std::monostate, SecurityFileFilter, SecurityNetworkFilter> config) {
4343
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_INFO,
4444
"[CreateFilterForCallname] EnableCallName:%s, idx:%ld, hold:%d \n",
45-
call_name.c_str(),
45+
callName.c_str(),
4646
config.index(),
4747
std::holds_alternative<SecurityFileFilter>(config));
4848
int ret = 0;
4949

50-
int call_name_idx = GetCallNameIdx(call_name);
51-
if (call_name_idx < 0)
52-
return 1;
50+
int callNameIdx = GetCallNameIdx(callName);
51+
if (callNameIdx == ERR_UNKNOWN_CALLNAME) {
52+
return ERR_DRIVER_INVALID_PARAM;
53+
}
5354

5455
auto filter = std::get_if<SecurityFileFilter>(&config);
5556
// update filters map
5657
std::vector<path_entry> path_entries;
5758
// concatenate path and filename, then write the resulting char* path into path_filter_list
5859
// TODO qianlu.kk use map in map feature to support filters for different call names
5960
if (filter && filter->mFilePathList.size()) {
60-
selector_filters kernel_filters;
61-
::memset(&kernel_filters, 0, sizeof(kernel_filters));
61+
selector_filters kernelFilters;
62+
::memset(&kernelFilters, 0, sizeof(kernelFilters));
6263

6364
int idx = IdAllocator::GetInstance()->GetNextId<StringPrefixMap>();
64-
if (idx < 0) {
65+
if (idx == ERR_LIMIT_EXCEEDED) {
6566
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN,
6667
"[CreateFilterForCallname][IDAllocator] Failed to get next id, reach max %d\n",
6768
IdAllocator::GetInstance()->GetMaxId<StringPrefixMap>());
68-
return 1;
69+
return ERR_DRIVER_INVALID_PARAM;
6970
}
7071
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN,
7172
"[CreateFilterForCallname] Get index %d for %s\n",
7273
idx,
73-
call_name.c_str());
74+
callName.c_str());
7475
// step1: add a new entry into string_prefix_maps, and assign a filter id
7576
// step2: add a filter into filter map and record filter type and filter id
76-
selector_filter k_filter;
77-
::memset(&k_filter, 0, sizeof(k_filter));
78-
k_filter.filter_type = FILTER_TYPE_FILE_PREFIX;
79-
k_filter.map_idx[0] = idx;
77+
selector_filter kFilter;
78+
::memset(&kFilter, 0, sizeof(kFilter));
79+
kFilter.filter_type = FILTER_TYPE_FILE_PREFIX;
80+
kFilter.map_idx[0] = idx;
8081
// in bytes
81-
// k_filter.vallen = x.length();
82-
kernel_filters.filter_count = 1;
83-
kernel_filters.filters[0] = k_filter;
82+
// kFilter.vallen = x.length();
83+
kernelFilters.filter_count = 1;
84+
kernelFilters.filters[0] = kFilter;
8485

8586
// LOG(INFO) << "filter not empty!";
8687
for (int i = 0; i < (int)filter->mFilePathList.size() && i < MAX_FILTER_FOR_PER_CALLNAME; i++) {
@@ -92,70 +93,71 @@ int CreateFileFilterForCallname(std::shared_ptr<logtail::ebpf::BPFWrapper<securi
9293
x.c_str());
9394

9495
// update inner map
95-
string_prefix_lpm_trie prefix_trie;
96-
::memset(&prefix_trie, 0, sizeof(prefix_trie));
97-
::memcpy(prefix_trie.data, x.data(), x.length());
98-
prefix_trie.prefixlen = x.length() * 8; // in bits
96+
string_prefix_lpm_trie prefixTrie;
97+
::memset(&prefixTrie, 0, sizeof(prefixTrie));
98+
::memcpy(prefixTrie.data, x.data(), x.length());
99+
prefixTrie.prefixlen = x.length() * 8; // in bits
99100
uint8_t val = 1;
100101
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN,
101102
"[CreateFilterForCallname][before update] prefix trie data: %s prefix_len: %u\n",
102-
prefix_trie.data,
103-
prefix_trie.prefixlen);
103+
prefixTrie.data,
104+
prefixTrie.prefixlen);
104105
ret = wrapper->UpdateInnerMapElem<StringPrefixMap>(
105-
std::string("string_prefix_maps"), &idx, &prefix_trie, &val, 0);
106+
std::string("string_prefix_maps"), &idx, &prefixTrie, &val, 0);
106107
if (ret) {
107108
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN,
108109
"[CreateFilterForCallname][update failed] prefix trie data: %s prefix_len: %u\n",
109-
prefix_trie.data,
110-
prefix_trie.prefixlen);
110+
prefixTrie.data,
111+
prefixTrie.prefixlen);
111112
continue;
112113
}
113114
}
114115

115116
// udpate filter_map
116-
wrapper->UpdateBPFHashMap("filter_map", &call_name_idx, &kernel_filters, 0);
117+
wrapper->UpdateBPFHashMap("filter_map", &callNameIdx, &kernelFilters, 0);
117118
}
118119

119120
return ret;
120121
}
121122

122123
int DeleteFileFilterForCallname(std::shared_ptr<logtail::ebpf::BPFWrapper<security_bpf>> wrapper,
123-
const std::string& call_name) {
124-
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN, "DeleteFilterForCallname %s\n", call_name.c_str());
125-
int call_name_idx = GetCallNameIdx(call_name);
126-
if (call_name_idx < 0)
127-
return 1;
124+
const std::string& callName) {
125+
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN, "DeleteFilterForCallname %s\n", callName.c_str());
126+
int callNameIdx = GetCallNameIdx(callName);
127+
if (callNameIdx == ERR_UNKNOWN_CALLNAME) {
128+
return ERR_DRIVER_INVALID_PARAM;
129+
}
128130
int ret = 0;
129131
// step1: detach callname
130132

131133
// step2: get filters for call name
132-
selector_filters kernel_filters;
133-
::memset(&kernel_filters, 0, sizeof(kernel_filters));
134+
selector_filters kernelFilters;
135+
::memset(&kernelFilters, 0, sizeof(kernelFilters));
134136
// get filters
135-
ret = wrapper->LookupBPFHashMap("filter_map", &call_name_idx, &kernel_filters);
137+
ret = wrapper->LookupBPFHashMap("filter_map", &callNameIdx, &kernelFilters);
136138
if (ret) {
137139
// no filters found, return directly
138140
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_INFO,
139141
"[DeleteFilterForCallname] there is no filter for call name: %s\n",
140-
call_name.c_str());
142+
callName.c_str());
141143
return 0;
142144
}
143145

144146
// step3: remove filters
145-
for (int i = 0; i < kernel_filters.filter_count; i++) {
146-
auto filter = kernel_filters.filters[i];
147-
auto outter_key = filter.map_idx[0];
148-
wrapper->DeleteInnerMap<StringPrefixMap>("string_prefix_maps", &outter_key);
149-
IdAllocator::GetInstance()->ReleaseId<StringPrefixMap>(outter_key);
147+
for (int i = 0; i < kernelFilters.filter_count; i++) {
148+
auto filter = kernelFilters.filters[i];
149+
auto outterKey = filter.map_idx[0];
150+
wrapper->DeleteInnerMap<StringPrefixMap>("string_prefix_maps", &outterKey);
151+
IdAllocator::GetInstance()->ReleaseId<StringPrefixMap>(outterKey);
150152
ebpf_log(logtail::ebpf::eBPFLogType::NAMI_LOG_TYPE_WARN,
151153
"[DeleteFilterForCallname] release filter for type: %d mapIdx: %u\n",
152154
static_cast<int>(filter.filter_type),
153-
outter_key);
155+
outterKey);
154156
}
155157

156158
// step4: delete filter map for call name
157-
::memset(&kernel_filters, 0, sizeof(kernel_filters));
158-
ret = wrapper->UpdateBPFHashMap("filter_map", &call_name_idx, &kernel_filters, 0);
159+
::memset(&kernelFilters, 0, sizeof(kernelFilters));
160+
ret = wrapper->UpdateBPFHashMap("filter_map", &callNameIdx, &kernelFilters, 0);
159161

160162
return ret;
161163
}

Diff for: ‎core/ebpf/driver/IdAllocator.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
namespace logtail {
2424
namespace ebpf {
2525

26+
#define ERR_LIMIT_EXCEEDED -1
27+
2628
template <typename BPFMap>
2729
class IdManager {
2830
public:
@@ -37,7 +39,7 @@ class IdManager {
3739
}
3840

3941
if (mNextId >= mIdMax) {
40-
return -1;
42+
return ERR_LIMIT_EXCEEDED;
4143
}
4244
return mNextId++;
4345
}

0 commit comments

Comments
 (0)