Skip to content

Commit 3a1b24b

Browse files
henryzhx8henryzhx8
authored
allow discardingnonutf8 in filter plugin for backward compatability (#2159)
1 parent 43a3f5f commit 3a1b24b

File tree

2 files changed

+137
-108
lines changed

2 files changed

+137
-108
lines changed

‎core/plugin/processor/ProcessorFilterNative.cpp

+80-108
Original file line numberDiff line numberDiff line change
@@ -30,142 +30,114 @@ const std::string ProcessorFilterNative::sName = "processor_filter_regex_native"
3030
bool ProcessorFilterNative::Init(const Json::Value& config) {
3131
std::string errorMsg;
3232

33-
// FilterKey + FilterRegex
34-
std::vector<std::string> filterKeys, filterRegs;
35-
if (!GetOptionalListParam(config, "FilterKey", filterKeys, errorMsg)
36-
|| !GetOptionalListParam(config, "FilterRegex", filterRegs, errorMsg)) {
37-
PARAM_WARNING_IGNORE(mContext->GetLogger(),
38-
mContext->GetAlarm(),
39-
errorMsg,
40-
sName,
41-
mContext->GetConfigName(),
42-
mContext->GetProjectName(),
43-
mContext->GetLogstoreName(),
44-
mContext->GetRegion());
45-
} else if (filterKeys.size() != filterRegs.size()) {
46-
PARAM_WARNING_IGNORE(mContext->GetLogger(),
47-
mContext->GetAlarm(),
48-
"param FilterKey and FilterRegex does not have the same size",
49-
sName,
50-
mContext->GetConfigName(),
51-
mContext->GetProjectName(),
52-
mContext->GetLogstoreName(),
53-
mContext->GetRegion());
54-
} else if (!filterKeys.empty()) {
55-
bool hasError = false;
56-
std::vector<boost::regex> regs;
57-
for (const auto& reg : filterRegs) {
58-
if (!IsRegexValid(reg)) {
59-
PARAM_WARNING_IGNORE(mContext->GetLogger(),
60-
mContext->GetAlarm(),
61-
"value in list param FilterRegex is not a valid regex",
62-
sName,
63-
mContext->GetConfigName(),
64-
mContext->GetProjectName(),
65-
mContext->GetLogstoreName(),
66-
mContext->GetRegion());
67-
hasError = true;
68-
break;
69-
}
70-
regs.emplace_back(reg);
71-
}
72-
if (!hasError) {
73-
mFilterRule = std::make_shared<LogFilterRule>();
74-
mFilterRule->FilterKeys = filterKeys;
75-
mFilterRule->FilterRegs = regs;
76-
mFilterMode = Mode::RULE_MODE;
77-
}
78-
}
79-
80-
// TODO: deprecated, remove in the future
81-
// Include
82-
if (mFilterMode == Mode::BYPASS_MODE) {
83-
if (!GetOptionalMapParam(config, "Include", mInclude, errorMsg)) {
84-
PARAM_WARNING_IGNORE(mContext->GetLogger(),
85-
mContext->GetAlarm(),
86-
errorMsg,
87-
sName,
88-
mContext->GetConfigName(),
89-
mContext->GetProjectName(),
90-
mContext->GetLogstoreName(),
91-
mContext->GetRegion());
92-
} else if (!mInclude.empty()) {
93-
std::vector<std::string> keys;
94-
std::vector<boost::regex> regs;
95-
bool hasError = false;
96-
for (auto& include : mInclude) {
97-
if (!IsRegexValid(include.second)) {
98-
PARAM_WARNING_IGNORE(mContext->GetLogger(),
99-
mContext->GetAlarm(),
100-
"value in map param Include is not a valid regex",
101-
sName,
102-
mContext->GetConfigName(),
103-
mContext->GetProjectName(),
104-
mContext->GetLogstoreName(),
105-
mContext->GetRegion());
106-
hasError = true;
107-
break;
108-
}
109-
keys.emplace_back(include.first);
110-
regs.emplace_back(boost::regex(include.second));
111-
}
112-
if (!hasError) {
113-
mFilterRule = std::make_shared<LogFilterRule>();
114-
mFilterRule->FilterKeys = keys;
115-
mFilterRule->FilterRegs = regs;
116-
mFilterMode = Mode::RULE_MODE;
117-
}
118-
}
119-
}
120-
33+
// for backward compatibility, ConditionExp prioritize over FilterKey and FilterRegex
12134
// ConditionExp
122-
if (mFilterMode == Mode::BYPASS_MODE) {
123-
const char* key = "ConditionExp";
124-
const Json::Value* itr = config.find(key, key + strlen(key));
125-
if (itr == nullptr) {
35+
const char* key = "ConditionExp";
36+
const Json::Value* itr = config.find(key, key + strlen(key));
37+
if (itr) {
38+
if (!itr->isObject()) {
12639
PARAM_ERROR_RETURN(mContext->GetLogger(),
12740
mContext->GetAlarm(),
128-
"param ConditionExp is missing",
41+
"object param ConditionExp is not of type object",
12942
sName,
13043
mContext->GetConfigName(),
13144
mContext->GetProjectName(),
13245
mContext->GetLogstoreName(),
13346
mContext->GetRegion());
13447
}
135-
if (!itr->isObject()) {
48+
BaseFilterNodePtr root = ParseExpressionFromJSON(*itr);
49+
if (!root) {
13650
PARAM_ERROR_RETURN(mContext->GetLogger(),
13751
mContext->GetAlarm(),
138-
"object param ConditionExp is not of type object",
52+
"object param ConditionExp is not valid",
13953
sName,
14054
mContext->GetConfigName(),
14155
mContext->GetProjectName(),
14256
mContext->GetLogstoreName(),
14357
mContext->GetRegion());
14458
}
145-
BaseFilterNodePtr root = ParseExpressionFromJSON(*itr);
146-
if (!root) {
59+
mConditionExp.swap(root);
60+
mFilterMode = Mode::EXPRESSION_MODE;
61+
}
62+
63+
if (mFilterMode == Mode::BYPASS_MODE) {
64+
// FilterKey + FilterRegex
65+
std::vector<std::string> filterKeys, filterRegs;
66+
if (!GetOptionalListParam(config, "FilterKey", filterKeys, errorMsg)
67+
|| !GetOptionalListParam(config, "FilterRegex", filterRegs, errorMsg)) {
14768
PARAM_ERROR_RETURN(mContext->GetLogger(),
14869
mContext->GetAlarm(),
149-
"object param ConditionExp is not valid",
70+
errorMsg,
71+
sName,
72+
mContext->GetConfigName(),
73+
mContext->GetProjectName(),
74+
mContext->GetLogstoreName(),
75+
mContext->GetRegion());
76+
} else if (filterKeys.size() != filterRegs.size()) {
77+
PARAM_ERROR_RETURN(mContext->GetLogger(),
78+
mContext->GetAlarm(),
79+
"param FilterKey and FilterRegex does not have the same size",
15080
sName,
15181
mContext->GetConfigName(),
15282
mContext->GetProjectName(),
15383
mContext->GetLogstoreName(),
15484
mContext->GetRegion());
85+
} else if (!filterKeys.empty()) {
86+
std::vector<boost::regex> regs;
87+
for (const auto& reg : filterRegs) {
88+
if (!IsRegexValid(reg)) {
89+
PARAM_ERROR_RETURN(mContext->GetLogger(),
90+
mContext->GetAlarm(),
91+
"value in list param FilterRegex is not a valid regex",
92+
sName,
93+
mContext->GetConfigName(),
94+
mContext->GetProjectName(),
95+
mContext->GetLogstoreName(),
96+
mContext->GetRegion());
97+
}
98+
regs.emplace_back(reg);
99+
}
100+
mFilterRule = std::make_shared<LogFilterRule>();
101+
mFilterRule->FilterKeys = filterKeys;
102+
mFilterRule->FilterRegs = regs;
103+
mFilterMode = Mode::RULE_MODE;
155104
}
156-
mConditionExp.swap(root);
157-
mFilterMode = Mode::EXPRESSION_MODE;
158105
}
159106

160107
if (mFilterMode == Mode::BYPASS_MODE) {
161-
PARAM_ERROR_RETURN(mContext->GetLogger(),
162-
mContext->GetAlarm(),
163-
"neither param Include nor param ConditionExp is valid",
164-
sName,
165-
mContext->GetConfigName(),
166-
mContext->GetProjectName(),
167-
mContext->GetLogstoreName(),
168-
mContext->GetRegion());
108+
// TODO: deprecated, remove in the future
109+
// Include
110+
if (!GetOptionalMapParam(config, "Include", mInclude, errorMsg)) {
111+
PARAM_ERROR_RETURN(mContext->GetLogger(),
112+
mContext->GetAlarm(),
113+
errorMsg,
114+
sName,
115+
mContext->GetConfigName(),
116+
mContext->GetProjectName(),
117+
mContext->GetLogstoreName(),
118+
mContext->GetRegion());
119+
} else if (!mInclude.empty()) {
120+
std::vector<std::string> keys;
121+
std::vector<boost::regex> regs;
122+
for (auto& include : mInclude) {
123+
if (!IsRegexValid(include.second)) {
124+
PARAM_ERROR_RETURN(mContext->GetLogger(),
125+
mContext->GetAlarm(),
126+
"value in map param Include is not a valid regex",
127+
sName,
128+
mContext->GetConfigName(),
129+
mContext->GetProjectName(),
130+
mContext->GetLogstoreName(),
131+
mContext->GetRegion());
132+
}
133+
keys.emplace_back(include.first);
134+
regs.emplace_back(boost::regex(include.second));
135+
}
136+
mFilterRule = std::make_shared<LogFilterRule>();
137+
mFilterRule->FilterKeys = keys;
138+
mFilterRule->FilterRegs = regs;
139+
mFilterMode = Mode::RULE_MODE;
140+
}
169141
}
170142

171143
// DiscardingNonUTF8

‎core/unittest/processor/ProcessorFilterNativeUnittest.cpp

+57
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,33 @@ void ProcessorFilterNativeUnittest::OnSuccessfulInit() {
7070
APSARA_TEST_TRUE(processor->Init(configJson));
7171
APSARA_TEST_EQUAL(1, processor->mFilterRule->FilterKeys.size());
7272
APSARA_TEST_EQUAL(1, processor->mFilterRule->FilterRegs.size());
73+
74+
// DiscardingNonUTF8
75+
configStr = R"(
76+
{
77+
"Type": "processor_filter_regex_native",
78+
"DiscardingNonUTF8": true
79+
}
80+
)";
81+
APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg));
82+
processor.reset(new ProcessorFilterNative());
83+
processor->SetContext(mContext);
84+
processor->SetMetricsRecordRef(ProcessorFilterNative::sName, "1");
85+
APSARA_TEST_TRUE(processor->Init(configJson));
86+
APSARA_TEST_TRUE(processor->mDiscardingNonUTF8);
87+
88+
configStr = R"(
89+
{
90+
"Type": "processor_filter_regex_native",
91+
"DiscardingNonUTF8": "true"
92+
}
93+
)";
94+
APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg));
95+
processor.reset(new ProcessorFilterNative());
96+
processor->SetContext(mContext);
97+
processor->SetMetricsRecordRef(ProcessorFilterNative::sName, "1");
98+
APSARA_TEST_TRUE(processor->Init(configJson));
99+
APSARA_TEST_FALSE(processor->mDiscardingNonUTF8);
73100
}
74101

75102
void ProcessorFilterNativeUnittest::OnFailedInit() {
@@ -78,6 +105,36 @@ void ProcessorFilterNativeUnittest::OnFailedInit() {
78105
string configStr, errorMsg;
79106

80107
// FilterKey + FilterRegex
108+
configStr = R"(
109+
{
110+
"Type": "processor_filter_regex_native",
111+
"FilterKey": true,
112+
"FilterRegex": true
113+
}
114+
)";
115+
APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg));
116+
processor.reset(new ProcessorFilterNative());
117+
processor->SetContext(mContext);
118+
processor->SetMetricsRecordRef(ProcessorFilterNative::sName, "1");
119+
APSARA_TEST_FALSE(processor->Init(configJson));
120+
121+
configStr = R"(
122+
{
123+
"Type": "processor_filter_regex_native",
124+
"FilterKey": [
125+
true
126+
],
127+
"FilterRegex": [
128+
"b"
129+
]
130+
}
131+
)";
132+
APSARA_TEST_TRUE(ParseJsonTable(configStr, configJson, errorMsg));
133+
processor.reset(new ProcessorFilterNative());
134+
processor->SetContext(mContext);
135+
processor->SetMetricsRecordRef(ProcessorFilterNative::sName, "1");
136+
APSARA_TEST_FALSE(processor->Init(configJson));
137+
81138
configStr = R"(
82139
{
83140
"Type": "processor_filter_regex_native",

0 commit comments

Comments
 (0)