Skip to content

Commit 77ae976

Browse files
chengshuyichengshuyi
authored
network observer supports mysql protocol (#2447)
* support mysql protocol Signed-off-by: chengshuyi <chengshuyi@linux.alibaba.com> * compilation success Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * code format Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * add test case Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * export mysql metric, log and span Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * add mysql dataelement Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * do some minor fix Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * add license into MysqlParser.h Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * remove comment on private tag Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * add MysqlRecord test case Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * fix private bug Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * remove iostream and modify license Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> * convert SQL to sql Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com> --------- Signed-off-by: chengshuyi <chengshuyi@linux.alibaba.com> Signed-off-by: Shuyi Cheng <chengshuyi@linux.alibaba.com>
1 parent d3e614a commit 77ae976

File tree

14 files changed

+524
-60
lines changed

14 files changed

+524
-60
lines changed

‎core/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ if (LINUX)
158158
if (ENABLE_ENTERPRISE)
159159
set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} shennong shennong/sdk apm/forward)
160160
endif()
161-
set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} ebpf ebpf/type ebpf/type/table ebpf/util ebpf/util/sampler ebpf/protocol/http ebpf/protocol ebpf/plugin/file_security ebpf/plugin/network_observer ebpf/plugin/process_security ebpf/plugin/network_security ebpf/plugin ebpf/observer ebpf/security
161+
set(SUB_DIRECTORIES_LIST ${SUB_DIRECTORIES_LIST} ebpf ebpf/type ebpf/type/table ebpf/util ebpf/util/sampler ebpf/protocol/http ebpf/protocol/mysql ebpf/protocol ebpf/plugin/file_security ebpf/plugin/network_observer ebpf/plugin/process_security ebpf/plugin/network_security ebpf/plugin ebpf/observer ebpf/security
162162
prometheus prometheus/labels prometheus/schedulers prometheus/async prometheus/component
163163
host_monitor host_monitor/collector host_monitor/common forward forward/loongsuite
164164
)

‎core/ebpf/plugin/network_observer/Connection.cpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,10 @@ namespace logtail::ebpf {
3434
static constexpr StringView kExternalStr = "external";
3535
static constexpr StringView kLocalhostStr = "localhost";
3636
static constexpr StringView kHttpStr = "http";
37+
static constexpr StringView kMysqlStr = "mysql";
38+
static constexpr StringView kSqlStr = "sql";
3739
static constexpr StringView kRpc25Str = "25";
40+
static constexpr StringView kRpc60Str = "60";
3841
static constexpr StringView kRpc0Str = "0";
3942
static constexpr StringView kHttpClientStr = "http_client";
4043
static constexpr StringView kUnknownStr = "unknown";
@@ -124,6 +127,11 @@ void Connection::TryAttachL7Meta(support_role_e role, support_proto_e protocol)
124127
mTags.SetNoCopy<kCallType>(kHttpStr);
125128
MarkL7MetaAttached();
126129
}
130+
} else if (mProtocol == support_proto_e::ProtoMySQL) {
131+
mTags.SetNoCopy<kRpcType>(kRpc60Str);
132+
mTags.SetNoCopy<kCallKind>(kSqlStr);
133+
mTags.SetNoCopy<kCallType>(kMysqlStr);
134+
MarkL7MetaAttached();
127135
}
128136
}
129137

‎core/ebpf/plugin/network_observer/Connection.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ class Connection {
6161
void UpdateConnState(struct conn_ctrl_event_t* event, bool& isClose);
6262

6363
[[nodiscard]] size_t GetContainerIdKey() const { return mCidKey; }
64+
[[nodiscard]] support_proto_e GetProtocol() const { return mProtocol; }
6465

6566
const StaticDataRow<&kConnTrackerTable>& GetConnTrackerAttrs() { return mTags; }
6667

@@ -191,8 +192,6 @@ class Connection {
191192
void MarkL4MetaAttached() { mMetaFlags.fetch_or(kSFlagL4MetaAttached, std::memory_order_release); }
192193
void MarkL7MetaAttached() { mMetaFlags.fetch_or(kSFlagL7MetaAttached, std::memory_order_release); }
193194

194-
[[nodiscard]] support_proto_e GetProtocol() const { return mProtocol; }
195-
196195
void MarkClose() {
197196
this->mIsClose = true;
198197
this->mMarkCloseTime = std::chrono::steady_clock::now();

‎core/ebpf/plugin/network_observer/NetworkObserverManager.cpp‎

Lines changed: 44 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -571,18 +571,26 @@ bool NetworkObserverManager::ConsumeLogAggregateTree() { // handler
571571
logEvent->SetContentNoCopy(kConnTrackerTable.ColLogKey(i), ctAttrVal[i]);
572572
}
573573
// set time stamp
574-
auto* httpRecord = static_cast<HttpRecord*>(record);
575-
auto timeSpec = ConvertKernelTimeToUnixTime(httpRecord->GetStartTimeStamp());
574+
logEvent->SetContent(kLatencyNS.LogKey(), std::to_string(record->GetLatencyNs()));
575+
auto timeSpec = ConvertKernelTimeToUnixTime(record->GetStartTimeStamp());
576576
logEvent->SetTimestamp(timeSpec.tv_sec, timeSpec.tv_nsec);
577-
logEvent->SetContent(kLatencyNS.LogKey(), std::to_string(httpRecord->GetLatencyNs()));
578-
logEvent->SetContent(kHTTPMethod.LogKey(), httpRecord->GetMethod());
579-
logEvent->SetContent(kHTTPPath.LogKey(),
580-
httpRecord->GetRealPath().size() ? httpRecord->GetRealPath()
581-
: httpRecord->GetPath());
582-
logEvent->SetContent(kHTTPVersion.LogKey(), httpRecord->GetProtocolVersion());
583-
logEvent->SetContent(kStatusCode.LogKey(), std::to_string(httpRecord->GetStatusCode()));
584-
logEvent->SetContent(kHTTPReqBody.LogKey(), httpRecord->GetReqBody());
585-
logEvent->SetContent(kHTTPRespBody.LogKey(), httpRecord->GetRespBody());
577+
auto protocol = record->GetConnection()->GetProtocol();
578+
if (protocol == support_proto_e::ProtoHTTP) {
579+
auto* httpRecord = static_cast<HttpRecord*>(record);
580+
logEvent->SetContent(kHTTPMethod.LogKey(), httpRecord->GetMethod());
581+
logEvent->SetContent(kHTTPPath.LogKey(),
582+
httpRecord->GetRealPath().size() ? httpRecord->GetRealPath()
583+
: httpRecord->GetPath());
584+
logEvent->SetContent(kHTTPVersion.LogKey(), httpRecord->GetProtocolVersion());
585+
logEvent->SetContent(kStatusCode.LogKey(), std::to_string(httpRecord->GetStatusCode()));
586+
logEvent->SetContent(kHTTPReqBody.LogKey(), httpRecord->GetReqBody());
587+
logEvent->SetContent(kHTTPRespBody.LogKey(), httpRecord->GetRespBody());
588+
} else if (protocol == support_proto_e::ProtoMySQL) {
589+
auto* mysqlRecord = static_cast<MysqlRecord*>(record);
590+
logEvent->SetContent(kDBSystemName.LogKey(), "mysql");
591+
logEvent->SetContent(kDBResponseStatusCode.LogKey(), std::to_string(mysqlRecord->GetStatusCode()));
592+
logEvent->SetContent(kDBStatement.LogKey(), mysqlRecord->GetSql());
593+
}
586594
LOG_DEBUG(sLogger, ("add one log, log timestamp", timeSpec.tv_sec)("nano", timeSpec.tv_nsec));
587595
needPush = true;
588596
}
@@ -1072,19 +1080,26 @@ bool NetworkObserverManager::ConsumeSpanAggregateTree() { // handler
10721080
}
10731081

10741082
spanEvent->SetName(record->GetSpanName());
1075-
auto* httpRecord = static_cast<HttpRecord*>(record);
1076-
spanEvent->SetTag(kRpc.SpanKey(), httpRecord->GetConvSpanName());
1077-
if (!ct->IsServer()) {
1078-
spanEvent->SetTag(kEndpoint.SpanKey(), httpRecord->GetConvSpanName());
1083+
auto protocol = record->GetConnection()->GetProtocol();
1084+
if (protocol == support_proto_e::ProtoHTTP) {
1085+
auto* httpRecord = static_cast<HttpRecord*>(record);
1086+
spanEvent->SetTag(kRpc.SpanKey(), httpRecord->GetConvSpanName());
1087+
if (!ct->IsServer()) {
1088+
spanEvent->SetTag(kEndpoint.SpanKey(), httpRecord->GetConvSpanName());
1089+
}
1090+
spanEvent->SetTag(kHTTPReqBody.SpanKey(), httpRecord->GetReqBody());
1091+
spanEvent->SetTag(kHTTPRespBody.SpanKey(), httpRecord->GetRespBody());
1092+
spanEvent->SetTag(kHTTPReqBodySize.SpanKey(), std::to_string(httpRecord->GetReqBodySize()));
1093+
spanEvent->SetTag(kHTTPRespBodySize.SpanKey(), std::to_string(httpRecord->GetRespBodySize()));
1094+
spanEvent->SetTag(kHTTPVersion.SpanKey(), httpRecord->GetProtocolVersion());
1095+
// spanEvent->SetTag(kHTTPReqHeader.SpanKey(), httpRecord->GetReqHeaderMap());
1096+
// spanEvent->SetTag(kHTTPRespHeader.SpanKey(), httpRecord->GetRespHeaders());
1097+
} else if (protocol == support_proto_e::ProtoMySQL) {
1098+
auto* mysqlRecord = static_cast<MysqlRecord*>(record);
1099+
spanEvent->SetTag(kDBSystemName.SpanKey(), "mysql");
1100+
spanEvent->SetTag(kDBResponseStatusCode.SpanKey(), std::to_string(mysqlRecord->GetStatusCode()));
1101+
spanEvent->SetTag(kDBStatement.SpanKey(), mysqlRecord->GetSql());
10791102
}
1080-
spanEvent->SetTag(kHTTPReqBody.SpanKey(), httpRecord->GetReqBody());
1081-
spanEvent->SetTag(kHTTPRespBody.SpanKey(), httpRecord->GetRespBody());
1082-
spanEvent->SetTag(kHTTPReqBodySize.SpanKey(), std::to_string(httpRecord->GetReqBodySize()));
1083-
spanEvent->SetTag(kHTTPRespBodySize.SpanKey(), std::to_string(httpRecord->GetRespBodySize()));
1084-
spanEvent->SetTag(kHTTPVersion.SpanKey(), httpRecord->GetProtocolVersion());
1085-
1086-
// spanEvent->SetTag(kHTTPReqHeader.SpanKey(), httpRecord->GetReqHeaderMap());
1087-
// spanEvent->SetTag(kHTTPRespHeader.SpanKey(), httpRecord->GetRespHeaders());
10881103

10891104
struct timespec startTime = ConvertKernelTimeToUnixTime(record->GetStartTimeStamp());
10901105
struct timespec endTime = ConvertKernelTimeToUnixTime(record->GetEndTimeStamp());
@@ -1891,17 +1906,17 @@ void NetworkObserverManager::ReportAgentInfo() {
18911906
}
18921907

18931908
int NetworkObserverManager::HandleEvent([[maybe_unused]] const std::shared_ptr<CommonEvent>& commonEvent) {
1894-
auto* httpRecord = static_cast<HttpRecord*>(commonEvent.get());
1895-
if (httpRecord) {
1896-
auto appDetail = httpRecord->GetAppDetail();
1897-
if (appDetail->mEnableLog && httpRecord->ShouldSample()) {
1909+
auto* record = static_cast<L7Record*>(commonEvent.get());
1910+
if (record) {
1911+
auto appDetail = record->GetAppDetail();
1912+
if (appDetail->mEnableLog && record->ShouldSample()) {
18981913
processRecordAsLog(commonEvent, appDetail);
18991914
}
1900-
if (appDetail->mEnableSpan && httpRecord->ShouldSample()) {
1915+
if (appDetail->mEnableSpan && record->ShouldSample()) {
19011916
processRecordAsSpan(commonEvent, appDetail);
19021917
}
19031918
if (appDetail->mEnableMetric) {
1904-
processRecordAsMetric(httpRecord, appDetail);
1919+
processRecordAsMetric(record, appDetail);
19051920
}
19061921
}
19071922
return 0;

‎core/ebpf/protocol/AbstractParser.h‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,32 @@
2323

2424
namespace logtail::ebpf {
2525

26+
enum class ParseState {
27+
kUnknown,
28+
29+
// The parse failed: data is invalid.
30+
// Input buffer consumed is not consumed and parsed output element is invalid.
31+
kInvalid,
32+
33+
// The parse is partial: data appears to be an incomplete message.
34+
// Input buffer may be partially consumed and the parsed output element is not fully populated.
35+
kNeedsMoreData,
36+
37+
// The parse succeeded, but the data is ignored.
38+
// Input buffer is consumed, but the parsed output element is invalid.
39+
kIgnored,
40+
41+
// The parse succeeded, but indicated the end-of-stream.
42+
// Input buffer is consumed, and the parsed output element is valid.
43+
// however, caller should stop parsing any future data on this stream, even if more data exists.
44+
// Use cases include messages that indicate a change in protocol (see HTTP status 101).
45+
kEOS,
46+
47+
// The parse succeeded.
48+
// Input buffer is consumed, and the parsed output element is valid.
49+
kSuccess,
50+
};
51+
2652
class AbstractProtocolParser {
2753
public:
2854
virtual ~AbstractProtocolParser() = default;

‎core/ebpf/protocol/ProtocolParser.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,15 @@ extern "C" {
3131
namespace logtail::ebpf {
3232

3333
std::set<support_proto_e> ProtocolParserManager::AvaliableProtocolTypes() const {
34-
return {support_proto_e::ProtoHTTP};
34+
return {support_proto_e::ProtoHTTP, support_proto_e::ProtoMySQL};
3535
}
3636

3737
support_proto_e ProtocolStringToEnum(std::string protocol) {
3838
std::transform(protocol.begin(), protocol.end(), protocol.begin(), [](unsigned char c) { return std::toupper(c); });
3939
if (protocol == "HTTP") {
4040
return support_proto_e::ProtoHTTP;
41+
} else if (protocol == "MYSQL") {
42+
return support_proto_e::ProtoMySQL;
4143
}
4244

4345
return support_proto_e::ProtoUnknown;

‎core/ebpf/protocol/http/HttpParser.h‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -43,32 +43,6 @@ struct HTTPResponse {
4343
size_t mNumHeaders = kMaxNumHeaders;
4444
};
4545

46-
enum class ParseState {
47-
kUnknown,
48-
49-
// The parse failed: data is invalid.
50-
// Input buffer consumed is not consumed and parsed output element is invalid.
51-
kInvalid,
52-
53-
// The parse is partial: data appears to be an incomplete message.
54-
// Input buffer may be partially consumed and the parsed output element is not fully populated.
55-
kNeedsMoreData,
56-
57-
// The parse succeeded, but the data is ignored.
58-
// Input buffer is consumed, but the parsed output element is invalid.
59-
kIgnored,
60-
61-
// The parse succeeded, but indicated the end-of-stream.
62-
// Input buffer is consumed, and the parsed output element is valid.
63-
// however, caller should stop parsing any future data on this stream, even if more data exists.
64-
// Use cases include messages that indicate a change in protocol (see HTTP status 101).
65-
kEOS,
66-
67-
// The parse succeeded.
68-
// Input buffer is consumed, and the parsed output element is valid.
69-
kSuccess,
70-
};
71-
7246
namespace http {
7347

7448
ParseState ParseRequest(std::string_view& buf, std::shared_ptr<HttpRecord>& result, bool forceSample = false);

0 commit comments

Comments
 (0)