Skip to content

Commit b0e356a

Browse files
linrunqi08linrunqi08
authored
Use regexp2 to do container filter for golang docker stdout (#2508)
* use regexp2 to do container filter for golang docker stdout * fix comments * fix comments
1 parent 58cec75 commit b0e356a

File tree

11 files changed

+163
-101
lines changed

11 files changed

+163
-101
lines changed

‎core/unittest/container_manager/testDataSet/ContainerManagerUnittest/k8s_pod_regex_test.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,16 @@
123123
"expected_matched_ids": [
124124
"nginx-app-deployment"
125125
]
126+
},
127+
{
128+
"name": "k8s_pod_negative_lookahead",
129+
"description": "测试负向前瞻正则（匹配 nginx 开头但不是 nginx-web 的 Pod）",
130+
"filters": {
131+
"K8sPodRegex": "^nginx-(?!web).*"
132+
},
133+
"expected_matched_ids": [
134+
"nginx-app-deployment"
135+
]
126136
}
127137
]
128138
}

‎pkg/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ require (
99
github.com/cespare/xxhash/v2 v2.2.0
1010
github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575
1111
github.com/containerd/containerd v1.6.8
12+
github.com/dlclark/regexp2 v1.11.5
1213
github.com/docker/docker v28.1.1+incompatible
1314
github.com/go-kit/kit v0.12.0
1415
github.com/gofrs/uuid v4.2.0+incompatible

‎pkg/go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -373,6 +373,8 @@ github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8
373373
github.com/digitalocean/godo v1.58.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU=
374374
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
375375
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
376+
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
377+
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
376378
github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
377379
github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
378380
github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=

‎pkg/helper/containercenter/container_center.go‎

Lines changed: 73 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"sync/atomic"
3030
"time"
3131

32+
"github.com/dlclark/regexp2"
3233
"github.com/docker/docker/api/types/container"
3334
"github.com/docker/docker/api/types/events"
3435
"github.com/docker/docker/api/types/image"
@@ -74,13 +75,13 @@ type EnvConfigInfo struct {
7475

7576
// K8SFilter used for find specific container
7677
type K8SFilter struct {
77-
NamespaceReg *regexp.Regexp
78-
PodReg *regexp.Regexp
79-
ContainerReg *regexp.Regexp
78+
NamespaceReg *regexp2.Regexp
79+
PodReg *regexp2.Regexp
80+
ContainerReg *regexp2.Regexp
8081
IncludeLabels map[string]string
8182
ExcludeLabels map[string]string
82-
IncludeLabelRegs map[string]*regexp.Regexp
83-
ExcludeLabelRegs map[string]*regexp.Regexp
83+
IncludeLabelRegs map[string]*regexp2.Regexp
84+
ExcludeLabelRegs map[string]*regexp2.Regexp
8485
hashKey uint64
8586
}
8687

@@ -90,21 +91,21 @@ func CreateK8SFilter(ns, pod, container string, includeK8sLabels, excludeK8sLabe
9091
var err error
9192
var hashStrBuilder strings.Builder
9293
if len(ns) > 0 {
93-
if filter.NamespaceReg, err = regexp.Compile(ns); err != nil {
94+
if filter.NamespaceReg, err = regexp2.Compile(ns, 0); err != nil {
9495
return nil, err
9596
}
9697
}
9798
hashStrBuilder.WriteString(ns)
9899
hashStrBuilder.WriteString("$$$")
99100
if len(pod) > 0 {
100-
if filter.PodReg, err = regexp.Compile(pod); err != nil {
101+
if filter.PodReg, err = regexp2.Compile(pod, 0); err != nil {
101102
return nil, err
102103
}
103104
}
104105
hashStrBuilder.WriteString(pod)
105106
hashStrBuilder.WriteString("$$$")
106107
if len(container) > 0 {
107-
if filter.ContainerReg, err = regexp.Compile(container); err != nil {
108+
if filter.ContainerReg, err = regexp2.Compile(container, 0); err != nil {
108109
return nil, err
109110
}
110111
}
@@ -227,14 +228,32 @@ func (info *K8SInfo) IsMatch(filter *K8SFilter) bool {
227228

228229
// innerMatch ...
229230
func (info *K8SInfo) innerMatch(filter *K8SFilter) bool {
230-
if filter.NamespaceReg != nil && !filter.NamespaceReg.MatchString(info.Namespace) {
231-
return false
231+
if filter.NamespaceReg != nil {
232+
matched, err := filter.NamespaceReg.MatchString(info.Namespace)
233+
if err != nil {
234+
logger.Debug(context.Background(), "namespace regex match error", err, "namespace", info.Namespace, "pattern", filter.NamespaceReg.String())
235+
}
236+
if !matched {
237+
return false
238+
}
232239
}
233-
if filter.PodReg != nil && !filter.PodReg.MatchString(info.Pod) {
234-
return false
240+
if filter.PodReg != nil {
241+
matched, err := filter.PodReg.MatchString(info.Pod)
242+
if err != nil {
243+
logger.Debug(context.Background(), "pod regex match error", err, "pod", info.Pod, "pattern", filter.PodReg.String())
244+
}
245+
if !matched {
246+
return false
247+
}
235248
}
236-
if filter.ContainerReg != nil && !filter.ContainerReg.MatchString(info.ContainerName) {
237-
return false
249+
if filter.ContainerReg != nil {
250+
matched, err := filter.ContainerReg.MatchString(info.ContainerName)
251+
if err != nil {
252+
logger.Debug(context.Background(), "container regex match error", err, "container", info.ContainerName, "pattern", filter.ContainerReg.String())
253+
}
254+
if !matched {
255+
return false
256+
}
238257
}
239258
// if labels is nil, create an empty map
240259
if info.Labels == nil {
@@ -891,8 +910,8 @@ func (dc *ContainerCenter) setLastError(err error, msg string) {
891910

892911
func isMapLabelsMatch(includeLabel map[string]string,
893912
excludeLabel map[string]string,
894-
includeLabelRegex map[string]*regexp.Regexp,
895-
excludeLabelRegex map[string]*regexp.Regexp,
913+
includeLabelRegex map[string]*regexp2.Regexp,
914+
excludeLabelRegex map[string]*regexp2.Regexp,
896915
labels map[string]string) bool {
897916
if len(includeLabel) != 0 || len(includeLabelRegex) != 0 {
898917
matchedFlag := false
@@ -905,9 +924,15 @@ func isMapLabelsMatch(includeLabel map[string]string,
905924
// if matched, do not need check regex
906925
if !matchedFlag {
907926
for key, reg := range includeLabelRegex {
908-
if dockerVal, ok := labels[key]; ok && reg.MatchString(dockerVal) {
909-
matchedFlag = true
910-
break
927+
if dockerVal, ok := labels[key]; ok {
928+
matched, err := reg.MatchString(dockerVal)
929+
if err != nil {
930+
logger.Debug(context.Background(), "include label regex match error", err, "key", key, "value", dockerVal, "pattern", reg.String())
931+
}
932+
if matched {
933+
matchedFlag = true
934+
break
935+
}
911936
}
912937
}
913938
}
@@ -922,24 +947,30 @@ func isMapLabelsMatch(includeLabel map[string]string,
922947
}
923948
}
924949
for key, reg := range excludeLabelRegex {
925-
if dockerVal, ok := labels[key]; ok && reg.MatchString(dockerVal) {
926-
return false
950+
if dockerVal, ok := labels[key]; ok {
951+
matched, err := reg.MatchString(dockerVal)
952+
if err != nil {
953+
logger.Debug(context.Background(), "exclude label regex match error", err, "key", key, "value", dockerVal, "pattern", reg.String())
954+
}
955+
if matched {
956+
return false
957+
}
927958
}
928959
}
929960
return true
930961
}
931962

932963
func isContainerLabelMatch(includeLabel map[string]string,
933964
excludeLabel map[string]string,
934-
includeLabelRegex map[string]*regexp.Regexp,
935-
excludeLabelRegex map[string]*regexp.Regexp,
965+
includeLabelRegex map[string]*regexp2.Regexp,
966+
excludeLabelRegex map[string]*regexp2.Regexp,
936967
info *DockerInfoDetail) bool {
937968
return isMapLabelsMatch(includeLabel, excludeLabel, includeLabelRegex, excludeLabelRegex, info.ContainerInfo.Config.Labels)
938969
}
939970

940971
func isMathEnvItem(env string,
941972
staticEnv map[string]string,
942-
regexEnv map[string]*regexp.Regexp) bool {
973+
regexEnv map[string]*regexp2.Regexp) bool {
943974
var envKey, envValue string
944975
splitArray := strings.SplitN(env, "=", 2)
945976
if len(splitArray) < 2 {
@@ -956,17 +987,23 @@ func isMathEnvItem(env string,
956987
}
957988

958989
if len(regexEnv) > 0 {
959-
if reg, ok := regexEnv[envKey]; ok && reg.MatchString(envValue) {
960-
return true
990+
if reg, ok := regexEnv[envKey]; ok {
991+
matched, err := reg.MatchString(envValue)
992+
if err != nil {
993+
logger.Debug(context.Background(), "env regex match error", err, "key", envKey, "value", envValue, "pattern", reg.String())
994+
}
995+
if matched {
996+
return true
997+
}
961998
}
962999
}
9631000
return false
9641001
}
9651002

9661003
func isContainerEnvMatch(includeEnv map[string]string,
9671004
excludeEnv map[string]string,
968-
includeEnvRegex map[string]*regexp.Regexp,
969-
excludeEnvRegex map[string]*regexp.Regexp,
1005+
includeEnvRegex map[string]*regexp2.Regexp,
1006+
excludeEnvRegex map[string]*regexp2.Regexp,
9701007
info *DockerInfoDetail) bool {
9711008

9721009
if len(includeEnv) != 0 || len(includeEnvRegex) != 0 {
@@ -996,12 +1033,12 @@ func isContainerEnvMatch(includeEnv map[string]string,
9961033
func (dc *ContainerCenter) getAllAcceptedInfo(
9971034
includeLabel map[string]string,
9981035
excludeLabel map[string]string,
999-
includeLabelRegex map[string]*regexp.Regexp,
1000-
excludeLabelRegex map[string]*regexp.Regexp,
1036+
includeLabelRegex map[string]*regexp2.Regexp,
1037+
excludeLabelRegex map[string]*regexp2.Regexp,
10011038
includeEnv map[string]string,
10021039
excludeEnv map[string]string,
1003-
includeEnvRegex map[string]*regexp.Regexp,
1004-
excludeEnvRegex map[string]*regexp.Regexp,
1040+
includeEnvRegex map[string]*regexp2.Regexp,
1041+
excludeEnvRegex map[string]*regexp2.Regexp,
10051042
k8sFilter *K8SFilter,
10061043
) map[string]*DockerInfoDetail {
10071044
containerMap := make(map[string]*DockerInfoDetail)
@@ -1022,12 +1059,12 @@ func (dc *ContainerCenter) getAllAcceptedInfoV2(
10221059
matchList map[string]*DockerInfoDetail,
10231060
includeLabel map[string]string,
10241061
excludeLabel map[string]string,
1025-
includeLabelRegex map[string]*regexp.Regexp,
1026-
excludeLabelRegex map[string]*regexp.Regexp,
1062+
includeLabelRegex map[string]*regexp2.Regexp,
1063+
excludeLabelRegex map[string]*regexp2.Regexp,
10271064
includeEnv map[string]string,
10281065
excludeEnv map[string]string,
1029-
includeEnvRegex map[string]*regexp.Regexp,
1030-
excludeEnvRegex map[string]*regexp.Regexp,
1066+
includeEnvRegex map[string]*regexp2.Regexp,
1067+
excludeEnvRegex map[string]*regexp2.Regexp,
10311068
k8sFilter *K8SFilter,
10321069
) (newCount, delCount int, matchAddedList, matchDeletedList []string) {
10331070

‎pkg/helper/containercenter/container_export.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ package containercenter
1717
import (
1818
"context"
1919
"fmt"
20-
"regexp"
2120
"strings"
2221
"time"
2322

23+
"github.com/dlclark/regexp2"
2424
"github.com/docker/docker/api/types/container"
2525
"github.com/docker/docker/api/types/events"
2626
docker "github.com/docker/docker/client"
@@ -119,12 +119,12 @@ func GetContainerBySpecificInfo(filter func(*DockerInfoDetail) bool) (infoList [
119119
func GetContainerByAcceptedInfo(
120120
includeLabel map[string]string,
121121
excludeLabel map[string]string,
122-
includeLabelRegex map[string]*regexp.Regexp,
123-
excludeLabelRegex map[string]*regexp.Regexp,
122+
includeLabelRegex map[string]*regexp2.Regexp,
123+
excludeLabelRegex map[string]*regexp2.Regexp,
124124
includeEnv map[string]string,
125125
excludeEnv map[string]string,
126-
includeEnvRegex map[string]*regexp.Regexp,
127-
excludeEnvRegex map[string]*regexp.Regexp,
126+
includeEnvRegex map[string]*regexp2.Regexp,
127+
excludeEnvRegex map[string]*regexp2.Regexp,
128128
k8sFilter *K8SFilter,
129129
) map[string]*DockerInfoDetail {
130130
return getContainerCenterInstance().getAllAcceptedInfo(includeLabel, excludeLabel, includeLabelRegex, excludeLabelRegex, includeEnv, excludeEnv, includeEnvRegex, excludeEnvRegex, k8sFilter)
@@ -154,12 +154,12 @@ func GetContainerByAcceptedInfoV2(
154154
matchList map[string]*DockerInfoDetail,
155155
includeLabel map[string]string,
156156
excludeLabel map[string]string,
157-
includeLabelRegex map[string]*regexp.Regexp,
158-
excludeLabelRegex map[string]*regexp.Regexp,
157+
includeLabelRegex map[string]*regexp2.Regexp,
158+
excludeLabelRegex map[string]*regexp2.Regexp,
159159
includeEnv map[string]string,
160160
excludeEnv map[string]string,
161-
includeEnvRegex map[string]*regexp.Regexp,
162-
excludeEnvRegex map[string]*regexp.Regexp,
161+
includeEnvRegex map[string]*regexp2.Regexp,
162+
excludeEnvRegex map[string]*regexp2.Regexp,
163163
k8sFilter *K8SFilter,
164164
) (newCount, delCount int, matchAddedList, matchDeletedList []string) {
165165
return getContainerCenterInstance().getAllAcceptedInfoV2(
@@ -173,12 +173,12 @@ func GetDiffContainers(fullList map[string]struct{}) (fullAddedList, fullDeleted
173173

174174
// SplitRegexFromMap extract regex from user config
175175
// regex must begin with ^ and end with $(we only check ^)
176-
func SplitRegexFromMap(input map[string]string) (staticResult map[string]string, regexResult map[string]*regexp.Regexp, err error) {
176+
func SplitRegexFromMap(input map[string]string) (staticResult map[string]string, regexResult map[string]*regexp2.Regexp, err error) {
177177
staticResult = make(map[string]string)
178-
regexResult = make(map[string]*regexp.Regexp)
178+
regexResult = make(map[string]*regexp2.Regexp)
179179
for key, value := range input {
180180
if strings.HasPrefix(value, "^") {
181-
reg, err := regexp.Compile(value)
181+
reg, err := regexp2.Compile(value, 0)
182182
if err != nil {
183183
err = fmt.Errorf("key : %s, value : %s is not valid regex, err is %s", key, value, err.Error())
184184
return input, nil, err

‎pkg/helper/containercenter/container_matching_consistency_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ import (
1818
"encoding/json"
1919
"os"
2020
"path/filepath"
21-
"regexp"
2221
"sort"
2322
"testing"
2423

24+
"github.com/dlclark/regexp2"
2525
"github.com/docker/docker/api/types/container"
2626
"github.com/stretchr/testify/assert"
2727
"github.com/stretchr/testify/require"
@@ -101,7 +101,7 @@ func convertTestContainerToDockerInfoDetail(tc TestContainer) *DockerInfoDetail
101101
}
102102

103103
// convertTestFiltersToGoFilters converts test filters to Go filter structures
104-
func convertTestFiltersToGoFilters(tf TestFilters) (map[string]string, map[string]string, map[string]*regexp.Regexp, map[string]*regexp.Regexp, map[string]string, map[string]string, map[string]*regexp.Regexp, map[string]*regexp.Regexp, *K8SFilter) {
104+
func convertTestFiltersToGoFilters(tf TestFilters) (map[string]string, map[string]string, map[string]*regexp2.Regexp, map[string]*regexp2.Regexp, map[string]string, map[string]string, map[string]*regexp2.Regexp, map[string]*regexp2.Regexp, *K8SFilter) {
105105
// Container labels
106106
includeLabel, includeLabelRegex, _ := SplitRegexFromMap(tf.IncludeContainerLabel)
107107
excludeLabel, excludeLabelRegex, _ := SplitRegexFromMap(tf.ExcludeContainerLabel)
@@ -234,11 +234,11 @@ func TestContainerMatchingLogicValidation(t *testing.T) {
234234

235235
// Empty filters should match
236236
result := isContainerLabelMatch(map[string]string{}, map[string]string{},
237-
map[string]*regexp.Regexp{}, map[string]*regexp.Regexp{}, container)
237+
map[string]*regexp2.Regexp{}, map[string]*regexp2.Regexp{}, container)
238238
assert.True(t, result)
239239

240240
result = isContainerEnvMatch(map[string]string{}, map[string]string{},
241-
map[string]*regexp.Regexp{}, map[string]*regexp.Regexp{}, container)
241+
map[string]*regexp2.Regexp{}, map[string]*regexp2.Regexp{}, container)
242242
assert.True(t, result)
243243

244244
k8sFilter, _ := CreateK8SFilter("", "", "", map[string]string{}, map[string]string{})
@@ -259,7 +259,7 @@ func TestContainerMatchingLogicValidation(t *testing.T) {
259259

260260
// Include filter that doesn't match should fail
261261
result := isContainerLabelMatch(map[string]string{"nonexistent": "value"}, map[string]string{},
262-
map[string]*regexp.Regexp{}, map[string]*regexp.Regexp{}, container)
262+
map[string]*regexp2.Regexp{}, map[string]*regexp2.Regexp{}, container)
263263
assert.False(t, result)
264264
})
265265

@@ -275,7 +275,7 @@ func TestContainerMatchingLogicValidation(t *testing.T) {
275275

276276
// Exclude filter that matches should fail
277277
result := isContainerLabelMatch(map[string]string{}, map[string]string{"app": "test"},
278-
map[string]*regexp.Regexp{}, map[string]*regexp.Regexp{}, container)
278+
map[string]*regexp2.Regexp{}, map[string]*regexp2.Regexp{}, container)
279279
assert.False(t, result)
280280
})
281281

‎plugins/input/docker/rawstdout/input_docker_stdout.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"sync"
2424
"time"
2525

26+
"github.com/dlclark/regexp2"
2627
"github.com/docker/docker/api/types/container"
2728
docker "github.com/docker/docker/client"
2829
"github.com/docker/docker/pkg/stdcopy"
@@ -354,10 +355,10 @@ type ServiceDockerStdout struct {
354355
K8sContainerRegex string
355356

356357
// export from ilogtail-trace component
357-
IncludeLabelRegex map[string]*regexp.Regexp
358-
ExcludeLabelRegex map[string]*regexp.Regexp
359-
IncludeEnvRegex map[string]*regexp.Regexp
360-
ExcludeEnvRegex map[string]*regexp.Regexp
358+
IncludeLabelRegex map[string]*regexp2.Regexp
359+
ExcludeLabelRegex map[string]*regexp2.Regexp
360+
IncludeEnvRegex map[string]*regexp2.Regexp
361+
ExcludeEnvRegex map[string]*regexp2.Regexp
361362
K8sFilter *containercenter.K8SFilter
362363

363364
synerMap map[string]*stdoutSyner

0 commit comments

Comments
 (0)