Skip to content

Commit 6228b71

Browse files
mmetcmmetc
committed
Merge branch 'master' into releases/1.6.x
2 parents 8fc2205 + 02d4793 commit 6228b71

File tree

98 files changed

+147
-154
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

98 files changed

+147
-154
lines changed

Diff for: ‎.github/workflows/publish-tarball-release.yml

-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
# .github/workflows/build-docker-image.yml
21
name: Release
32

43
on:

Diff for: ‎.golangci.yml

-4
Original file line numberDiff line numberDiff line change
@@ -403,10 +403,6 @@ issues:
403403
path: pkg/acquisition/modules/victorialogs/internal/vlclient/vl_client.go
404404
text: "confusing-naming: Method 'QueryRange' differs only by capitalization to method 'queryRange' in the same source file"
405405

406-
- linters:
407-
- revive
408-
path: cmd/crowdsec-cli/copyfile.go
409-
410406
- linters:
411407
- revive
412408
path: pkg/hubtest/hubtest_item.go

Diff for: ‎config/crowdsec.cron.daily

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ test -x /usr/bin/cscli || exit 0
55
# splay hub upgrade and crowdsec reload
66
sleep "$(seq 1 300 | shuf -n 1)"
77

8-
/usr/bin/cscli --error hub update
8+
/usr/bin/cscli --error hub update >/dev/null
99

1010
upgraded=$(/usr/bin/cscli --error hub upgrade)
1111
if [ -n "$upgraded" ]; then

Diff for: ‎pkg/acquisition/modules/appsec/appsec.go

+11-8
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@ type AppsecSource struct {
6363
lapiURL string
6464
AuthCache AuthCache
6565
AppsecRunners []AppsecRunner // one for each go-routine
66-
apiClient *apiclient.ApiClient
6766
appsecAllowlistClient *allowlists.AppsecAllowlist
6867
}
6968

@@ -226,12 +225,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLe
226225

227226
w.AppsecRunners = make([]AppsecRunner, w.config.Routines)
228227

229-
w.apiClient, err = apiclient.GetLAPIClient()
230-
if err != nil {
231-
return fmt.Errorf("unable to get authenticated LAPI client: %w", err)
232-
}
233-
234-
w.appsecAllowlistClient = allowlists.NewAppsecAllowlist(w.apiClient, w.logger)
228+
w.appsecAllowlistClient = allowlists.NewAppsecAllowlist(w.logger)
235229

236230
for nbRoutine := range w.config.Routines {
237231
appsecRunnerUUID := uuid.New().String()
@@ -282,7 +276,16 @@ func (w *AppsecSource) OneShotAcquisition(_ context.Context, _ chan types.Event,
282276
func (w *AppsecSource) StreamingAcquisition(ctx context.Context, out chan types.Event, t *tomb.Tomb) error {
283277
w.outChan = out
284278

285-
w.appsecAllowlistClient.StartRefresh(t)
279+
apiClient, err := apiclient.GetLAPIClient()
280+
if err != nil {
281+
return fmt.Errorf("unable to get authenticated LAPI client: %w", err)
282+
}
283+
284+
err = w.appsecAllowlistClient.Start(ctx, apiClient)
285+
if err != nil {
286+
return fmt.Errorf("failed to fetch allowlists: %w", err)
287+
}
288+
w.appsecAllowlistClient.StartRefresh(ctx, t)
286289

287290
t.Go(func() error {
288291
defer trace.CatchPanic("crowdsec/acquis/appsec/live")

Diff for: ‎pkg/acquisition/modules/appsec/appsec_test.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -146,10 +146,9 @@ func loadAppSecEngine(test appsecRuleTest, t *testing.T) {
146146
assert.NoError(t, err)
147147
})
148148

149-
allowlistClient := allowlists.NewAppsecAllowlist(client, logger)
150-
// In real life, allowlists updater is started by the acquisition
151-
// Do it manually here as we are simulating the appsec itself
152-
err = allowlistClient.FetchAllowlists()
149+
allowlistClient := allowlists.NewAppsecAllowlist(logger)
150+
151+
err = allowlistClient.Start(t.Context(), client)
153152
require.NoError(t, err)
154153
runner := AppsecRunner{
155154
inChan: InChan,

Diff for: ‎pkg/acquisition/modules/syslog/syslog.go

+66-29
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package syslogacquisition
22

33
import (
4+
"bytes"
45
"context"
56
"errors"
67
"fmt"
@@ -27,6 +28,7 @@ type SyslogConfiguration struct {
2728
Port int `yaml:"listen_port,omitempty"`
2829
Addr string `yaml:"listen_addr,omitempty"`
2930
MaxMessageLen int `yaml:"max_message_len,omitempty"`
31+
DisableRFCParser bool `yaml:"disable_rfc_parser,omitempty"` // if true, we don't try to be smart and just remove the PRI
3032
configuration.DataSourceCommonCfg `yaml:",inline"`
3133
}
3234

@@ -182,6 +184,66 @@ func (s *SyslogSource) buildLogFromSyslog(ts time.Time, hostname string,
182184
return ret
183185
}
184186

187+
func (s *SyslogSource) parseLine(syslogLine syslogserver.SyslogMessage) string {
188+
var line string
189+
190+
logger := s.logger.WithField("client", syslogLine.Client)
191+
logger.Tracef("raw: %s", syslogLine)
192+
if s.metricsLevel != configuration.METRICS_NONE {
193+
linesReceived.With(prometheus.Labels{"source": syslogLine.Client}).Inc()
194+
}
195+
if !s.config.DisableRFCParser {
196+
p := rfc3164.NewRFC3164Parser(rfc3164.WithCurrentYear())
197+
err := p.Parse(syslogLine.Message)
198+
if err != nil {
199+
logger.Debugf("could not parse as RFC3164 (%s)", err)
200+
p2 := rfc5424.NewRFC5424Parser()
201+
err = p2.Parse(syslogLine.Message)
202+
if err != nil {
203+
logger.Errorf("could not parse message: %s", err)
204+
logger.Debugf("could not parse as RFC5424 (%s) : %s", err, syslogLine.Message)
205+
return ""
206+
}
207+
line = s.buildLogFromSyslog(p2.Timestamp, p2.Hostname, p2.Tag, p2.PID, p2.Message)
208+
if s.metricsLevel != configuration.METRICS_NONE {
209+
linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc5424"}).Inc()
210+
}
211+
} else {
212+
line = s.buildLogFromSyslog(p.Timestamp, p.Hostname, p.Tag, p.PID, p.Message)
213+
if s.metricsLevel != configuration.METRICS_NONE {
214+
linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc3164"}).Inc()
215+
}
216+
}
217+
} else {
218+
if len(syslogLine.Message) < 3 {
219+
logger.Errorf("malformated message, missing PRI (message too short)")
220+
return ""
221+
}
222+
if syslogLine.Message[0] != '<' {
223+
logger.Errorf("malformated message, missing PRI beginning")
224+
return ""
225+
}
226+
priEnd := bytes.Index(syslogLine.Message, []byte(">"))
227+
if priEnd == -1 {
228+
logger.Errorf("malformated message, missing PRI end")
229+
return ""
230+
}
231+
if priEnd > 4 {
232+
logger.Errorf("malformated message, PRI too long")
233+
return ""
234+
}
235+
for i := 1; i < priEnd; i++ {
236+
if syslogLine.Message[i] < '0' || syslogLine.Message[i] > '9' {
237+
logger.Errorf("malformated message, PRI not a number")
238+
return ""
239+
}
240+
}
241+
line = string(syslogLine.Message[priEnd+1:])
242+
}
243+
244+
return strings.TrimSuffix(line, "\n")
245+
}
246+
185247
func (s *SyslogSource) handleSyslogMsg(out chan types.Event, t *tomb.Tomb, c chan syslogserver.SyslogMessage) error {
186248
killed := false
187249
for {
@@ -196,37 +258,12 @@ func (s *SyslogSource) handleSyslogMsg(out chan types.Event, t *tomb.Tomb, c cha
196258
s.logger.Info("Syslog server has exited")
197259
return nil
198260
case syslogLine := <-c:
199-
var line string
200-
var ts time.Time
201-
202-
logger := s.logger.WithField("client", syslogLine.Client)
203-
logger.Tracef("raw: %s", syslogLine)
204-
if s.metricsLevel != configuration.METRICS_NONE {
205-
linesReceived.With(prometheus.Labels{"source": syslogLine.Client}).Inc()
206-
}
207-
p := rfc3164.NewRFC3164Parser(rfc3164.WithCurrentYear())
208-
err := p.Parse(syslogLine.Message)
209-
if err != nil {
210-
logger.Debugf("could not parse as RFC3164 (%s)", err)
211-
p2 := rfc5424.NewRFC5424Parser()
212-
err = p2.Parse(syslogLine.Message)
213-
if err != nil {
214-
logger.Errorf("could not parse message: %s", err)
215-
logger.Debugf("could not parse as RFC5424 (%s) : %s", err, syslogLine.Message)
216-
continue
217-
}
218-
line = s.buildLogFromSyslog(p2.Timestamp, p2.Hostname, p2.Tag, p2.PID, p2.Message)
219-
if s.metricsLevel != configuration.METRICS_NONE {
220-
linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc5424"}).Inc()
221-
}
222-
} else {
223-
line = s.buildLogFromSyslog(p.Timestamp, p.Hostname, p.Tag, p.PID, p.Message)
224-
if s.metricsLevel != configuration.METRICS_NONE {
225-
linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc3164"}).Inc()
226-
}
261+
line := s.parseLine(syslogLine)
262+
if line == "" {
263+
continue
227264
}
228265

229-
line = strings.TrimSuffix(line, "\n")
266+
var ts time.Time
230267

231268
l := types.Line{}
232269
l.Raw = line

Diff for: ‎pkg/acquisition/modules/syslog/syslog_test.go

+20
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,26 @@ listen_addr: 127.0.0.1`,
120120
`<13>May 18 12:37:56 mantis sshd`,
121121
},
122122
},
123+
{
124+
name: "RFC3164 - no parsing",
125+
config: `source: syslog
126+
listen_port: 4242
127+
listen_addr: 127.0.0.1
128+
disable_rfc_parser: true`,
129+
expectedLines: 5,
130+
logs: []string{
131+
`<13>May 18 12:37:56 mantis sshd[49340]: blabla2[foobar]`,
132+
`<13>May 18 12:37:56 mantis sshd[49340]: blabla2`,
133+
`<13>May 18 12:37:56 mantis sshd: blabla2`,
134+
`<13>May 18 12:37:56 mantis sshd`,
135+
`<999>May 18 12:37:56 mantis sshd`,
136+
`<1000>May 18 12:37:56 mantis sshd`,
137+
`>?> asd`,
138+
`<asd>asdasd`,
139+
`<1a asd`,
140+
`<123123>asdasd`,
141+
},
142+
},
123143
}
124144
if runtime.GOOS != "windows" {
125145
tests = append(tests, struct {

Diff for: ‎pkg/appsec/allowlists/allowlists.go

+23-16
Original file line numberDiff line numberDiff line change
@@ -36,25 +36,26 @@ type AppsecAllowlist struct {
3636
tomb *tomb.Tomb
3737
}
3838

39-
func NewAppsecAllowlist(client *apiclient.ApiClient, logger *log.Entry) *AppsecAllowlist {
39+
func NewAppsecAllowlist(logger *log.Entry) *AppsecAllowlist {
4040
a := &AppsecAllowlist{
41-
LAPIClient: client,
42-
logger: logger.WithField("component", "appsec-allowlist"),
43-
ips: []ipAllowlist{},
44-
ranges: []rangeAllowlist{},
45-
}
46-
47-
if err := a.FetchAllowlists(); err != nil {
48-
a.logger.Errorf("failed to fetch allowlists: %s", err)
41+
logger: logger.WithField("component", "appsec-allowlist"),
42+
ips: []ipAllowlist{},
43+
ranges: []rangeAllowlist{},
4944
}
5045

5146
return a
5247
}
5348

54-
func (a *AppsecAllowlist) FetchAllowlists() error {
49+
func (a *AppsecAllowlist) Start(ctx context.Context, client *apiclient.ApiClient) error {
50+
a.LAPIClient = client
51+
err := a.FetchAllowlists(ctx)
52+
return err
53+
}
54+
55+
func (a *AppsecAllowlist) FetchAllowlists(ctx context.Context) error {
5556
a.logger.Debug("fetching allowlists")
5657

57-
allowlists, _, err := a.LAPIClient.Allowlists.List(context.TODO(), apiclient.AllowlistListOpts{WithContent: true})
58+
allowlists, _, err := a.LAPIClient.Allowlists.List(ctx, apiclient.AllowlistListOpts{WithContent: true})
5859
if err != nil {
5960
return err
6061
}
@@ -92,32 +93,38 @@ func (a *AppsecAllowlist) FetchAllowlists() error {
9293
}
9394
}
9495

96+
if len(a.ips) != 0 || len(a.ranges) != 0 {
97+
a.logger.Infof("fetched %d IPs and %d ranges", len(a.ips), len(a.ranges))
98+
}
9599
a.logger.Debugf("fetched %d IPs and %d ranges", len(a.ips), len(a.ranges))
96100
a.logger.Tracef("allowlisted ips: %+v", a.ips)
97101
a.logger.Tracef("allowlisted ranges: %+v", a.ranges)
98102

99103
return nil
100104
}
101105

102-
func (a *AppsecAllowlist) updateAllowlists() error {
106+
func (a *AppsecAllowlist) updateAllowlists(ctx context.Context) {
103107
ticker := time.NewTicker(allowlistRefreshInterval)
104108

105109
for {
106110
select {
107111
case <-ticker.C:
108-
if err := a.FetchAllowlists(); err != nil {
112+
if err := a.FetchAllowlists(ctx); err != nil {
109113
a.logger.Errorf("failed to fetch allowlists: %s", err)
110114
}
111115
case <-a.tomb.Dying():
112116
ticker.Stop()
113-
return nil
117+
return
114118
}
115119
}
116120
}
117121

118-
func (a *AppsecAllowlist) StartRefresh(t *tomb.Tomb) {
122+
func (a *AppsecAllowlist) StartRefresh(ctx context.Context, t *tomb.Tomb) {
119123
a.tomb = t
120-
a.tomb.Go(a.updateAllowlists)
124+
a.tomb.Go(func() error {
125+
a.updateAllowlists(ctx)
126+
return nil
127+
})
121128
}
122129

123130
func (a *AppsecAllowlist) IsAllowlisted(sourceIP string) (bool, string) {

Diff for: ‎pkg/appsec/allowlists/allowlists_test.go

+7-3
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,13 @@ func TestAppsecAllowlist(t *testing.T) {
6464
assert.NoError(t, err)
6565
})
6666

67-
allowlistClient := NewAppsecAllowlist(client, log.NewEntry(log.StandardLogger()))
67+
ctx := t.Context()
68+
allowlistClient := NewAppsecAllowlist(log.NewEntry(log.StandardLogger()))
6869

69-
err = allowlistClient.FetchAllowlists()
70+
err = allowlistClient.Start(ctx, client)
71+
require.NoError(t, err)
72+
73+
err = allowlistClient.FetchAllowlists(ctx)
7074
require.NoError(t, err)
7175

7276
res, reason := allowlistClient.IsAllowlisted("1.2.3.4")
@@ -84,7 +88,7 @@ func TestAppsecAllowlist(t *testing.T) {
8488
assert.Len(t, allowlistClient.ips, 1)
8589
assert.Len(t, allowlistClient.ranges, 1)
8690

87-
err = allowlistClient.FetchAllowlists()
91+
err = allowlistClient.FetchAllowlists(ctx)
8892
require.NoError(t, err)
8993

9094
// No duplicates should be added

Diff for: ‎test/bats-detect/apache2-deb.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/apache2-rpm.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/asterisk-deb.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/asterisk-rpm.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/caddy-deb.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/caddy-rpm.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/dovecot-deb.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/dovecot-rpm.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/emby-deb.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

Diff for: ‎test/bats-detect/emby-rpm.bats

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/usr/bin/env bats
2-
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
32

43
set -u
54

0 commit comments

Comments
 (0)