Skip to content

feat: support custom regex for rule id lookup #628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
blinxen wants to merge 9 commits into
base: main
Choose a base branch
from
Open

feat: support custom regex for rule id lookup #628

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
152d505
feat: support custom regex for rule id lookup
blinxen Apr 20, 2026
682de7b
Fix mixed cause cause by copy-paste
blinxen Apr 20, 2026
224fcb7
Only overwrite rule id regexes on reset if they are not set
blinxen Apr 20, 2026
8154fdf
Return error if regex string could not be compiled instead of panicing
blinxen Apr 20, 2026
d791f62
Compare regex string instead of pointers in test
blinxen Apr 20, 2026
544b7a7
Improve doc comment
blinxen Apr 20, 2026
daec834
Use %w instead of %s to wrap errors
blinxen Apr 23, 2026
56e0226
fix: the project uses tabs instead of spaces
blinxen Apr 23, 2026
cad3ec0
Make sure there are at least two matched groups before continuing
blinxen Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ func NewDefaultConfig() *FTWConfiguration {
RunMode: DefaultRunMode,
MaxMarkerRetries: DefaultMaxMarkerRetries,
MaxMarkerLogLines: DefaultMaxMarkerLogLines,
StdLogIdRegex: DefaultStdLogIdRegex,
JsonLogIdRegex: DefaultJsonLogIdRegex,
}
}

Expand Down
4 changes: 4 additions & 0 deletions config/runner_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ type RunnerConfig struct {
// SkipTlsVerification skips certificate validation. Useful for connecting
// to domains with a self-signed certificate.
SkipTlsVerification bool
StdLogIdRegex string
JsonLogIdRegex string
}

type PlatformOverrides struct {
Expand All @@ -62,6 +64,8 @@ func NewRunnerConfiguration(cfg *FTWConfiguration) *RunnerConfig {
MaxMarkerRetries: cfg.MaxMarkerRetries,
RunMode: cfg.RunMode,
SkipTlsVerification: cfg.SkipTlsVerification,
StdLogIdRegex: cfg.StdLogIdRegex,
JsonLogIdRegex: cfg.JsonLogIdRegex,
}

if cfg.IncludeTests != nil {
Expand Down
18 changes: 18 additions & 0 deletions config/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,20 @@ const (
DefaultMaxMarkerRetries uint = 20
// DefaultMaxMarkerLogLines is the default lines we are going read back in a logfile to find the markers
DefaultMaxMarkerLogLines uint = 500
// DefaultStdLogIdRegex is the default regex used to look for rule IDs when reading the WAF logs
// Example "ids" that will be caught by this regex:
// - [id "999999"]
// - [id \"999999\"] (escaped quotes)
// - ["id":"999999"]
// - [\"id\":\"999999\"] (escaped quotes)
DefaultStdLogIdRegex string = `\[(?:id |\\?"id\\?":)\\?"(\d+)\\?"\]`
// DefaultJsonLogIdRegex is the default regex used to look for rule IDs when reading JSON WAF logs
// Example "ids" that will be caught by this regex:
// - {"id":4}
// - {..., "id":4,..}
// - {"ruleId":"4"}
// - {..., "ruleId":"4",...}
DefaultJsonLogIdRegex string = `(?:\{|,)\s*"(?:id|ruleId)":\s*"?(\d+)"?`
)

// FTWConfiguration FTW global Configuration
Expand All @@ -48,6 +62,10 @@ type FTWConfiguration struct {
IncludeTags *FTWRegexp `koanf:"include_tags"`
// to domains with a self-signed certificate.
SkipTlsVerification bool `koanf:"skip_tls_verification"`
// StdLogIdRegex is the regex used to look for rule IDs when reading the WAF logs
StdLogIdRegex string `koanf:"stdlogidregex"`
// JsonLogIdRegex is the same as StdLogIdRegex but used for parsing JSON logs
JsonLogIdRegex string `koanf:"jsonlogidregex"`
}

// FTWTestOverride holds four lists:
Expand Down
19 changes: 2 additions & 17 deletions waflog/read.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,6 @@ const maxRuleIdsEstimate = 15

var ruleIdsSet = make(map[uint]struct{}, maxRuleIdsEstimate)

// These regexes provide flexibility in parsing how the rule ID is logged.
// - [id "999999"]
// - [id \"999999\"] (escaped quotes)
// - ["id":"999999"]
// - [\"id\":\"999999\"] (escaped quotes)
var stdLogIdRegex = regexp.MustCompile(`\[(?:id |\\?"id\\?":)\\?"(\d+)\\?"\]`)

// - {"id":4}
// - {..., "id":4,..}
// - {"ruleId":"4"}
// - {..., "ruleId":"4",...}
var jsonLogIdRegex = regexp.MustCompile(`(?:\{|,)\s*"(?:id|ruleId)":\s*"?(\d+)"?`)

// TriggeredRules returns the IDs of all the rules found in the log for the current test
func (ll *FTWLogLines) TriggeredRules() []uint {
if ll.triggeredRulesInitialized {
Expand All @@ -44,11 +31,9 @@ func (ll *FTWLogLines) TriggeredRules() []uint {

for _, line := range lines {
log.Trace().Msgf("ftw/waflog: Looking for any rule in '%s'", line)
regex := stdLogIdRegex
match := regex.FindAllSubmatch(line, -1)
match := ll.stdLogIdRegex.FindAllSubmatch(line, -1)
if match == nil {
regex = jsonLogIdRegex
match = regex.FindAllSubmatch(line, -1)
match = ll.jsonLogIdRegex.FindAllSubmatch(line, -1)
}
for _, nextMatch := range match {
Comment thread
blinxen marked this conversation as resolved.
submatchBytes := nextMatch[1]
Expand Down
31 changes: 31 additions & 0 deletions waflog/read_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,3 +504,34 @@ func (s *readTestSuite) TestFalsePositiveIds() {
s.Len(foundRuleIds, 1)
s.Contains(foundRuleIds, uint(942370))
}

func (s *readTestSuite) TestFTWLogLines_CustomLogIdRegex() {
cfg, err := config.NewConfigFromEnv()
s.Require().NoError(err)
s.NotNil(cfg)

startMarker, endMarker := generateLogMarkers(100000, 1)
startMarkerLine := "X-cRs-TeSt: " + startMarker
endMarkerLine := "X-cRs-TeSt: " + endMarker
logLinesOnly :=
`[Tue Jan 05 02:21:09.637165 2021] [:error] [pid 76:tid 139683434571520] [client 172.23.0.1:58998] [client 172.23.0.1] ModSecurity: Warning. Pattern match "\\\\b(?:keep-alive|close),\\\\s?(?:keep-alive|close)\\\\b" at REQUEST_HEADERS:Connection. [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf"] [line "339"] [id "920210"] [msg "Multiple/Conflicting Connection Header Data Found"] [data "close,close"] [severity "WARNING"] [ver "OWASP_CRS/3.3.0"] [tag "application-multi"] [tag "language-multi"] [tag "platform-multi"] [tag "attack-protocol"] [tag "paranoia-level/1"] [tag "OWASP_CRS"] [tag "capec/1000/210/272"] [hostname "localhost"] [uri "/"] [unique_id "X-PNFSe1VwjCgYRI9FsbHgAAAIY"]
[Tue Jan 05 02:21:09.637731 2021] [:error] [pid 76:tid 139683434571520] [client 172.23.0.1:58998] [client 172.23.0.1] ModSecurity: Warning. Match of "pm AppleWebKit Android" against "REQUEST_HEADERS:User-Agent" required. [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-920-PROTOCOL-ENFORCEMENT.conf"] [line "1230"] [id="920300"] [msg "Request Missing an Accept Header"] [severity "NOTICE"] [ver "OWASP_CRS/3.3.0"] [tag "application-multi"] [tag "language-multi"] [tag "platform-multi"] [tag "attack-protocol"] [tag "OWASP_CRS"] [tag "capec/1000/210/272"] [tag "PCI/6.5.10"] [tag "paranoia-level/2"] [hostname "localhost"] [uri "/"] [unique_id "X-PNFSe1VwjCgYRI9FsbHgAAAIY"]
[Tue Jan 05 02:21:09.638572 2021] [:error] [pid 76:tid 139683434571520] [client 172.23.0.1:58998] [client 172.23.0.1] ModSecurity: Warning. Operator GE matched 5 at TX:anomaly_score. [file "/etc/modsecurity.d/owasp-crs/rules/REQUEST-949-BLOCKING-EVALUATION.conf"] [line "91"] [id "949110"] [msg "Inbound Anomaly Score Exceeded (Total Score: 5)"] [severity "CRITICAL"] [ver "OWASP_CRS/3.3.0"] [tag "application-multi"] [tag "language-multi"] [tag "platform-multi"] [tag "attack-generic"] [hostname "localhost"] [uri "/"] [unique_id "X-PNFSe1VwjCgYRI9FsbHgAAAIY"]`
logLines := fmt.Sprintf("%s\n%s\n%s", startMarkerLine, logLinesOnly, endMarkerLine)
s.filename, err = utils.CreateTempFileWithContent("", logLines, "test-errorlog-")
s.Require().NoError(err)

cfg.LogFile = s.filename
runnerConfig := config.NewRunnerConfiguration(cfg)

ll, err := NewFTWLogLines(runnerConfig)
s.Require().NoError(err)
ll.WithStartMarker(bytes.ToLower([]byte(startMarkerLine)))
ll.WithEndMarker(bytes.ToLower([]byte(endMarkerLine)))
err = ll.WithStdLogIdRegex(`\[id="(\d+)"\]`)
Copy link

Copilot AI Apr 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This line is indented with spaces and doesn’t match gofmt formatting used throughout the file; running gofmt will fix it and avoid style/tooling failures in CI/pre-commit hooks.

Suggested change
err = ll.WithStdLogIdRegex(`\[id="(\d+)"\]`)
err = ll.WithStdLogIdRegex(`\[id="(\d+)"\]`)

Copilot uses AI. Check for mistakes.
Copy link
Copy Markdown
Author

@blinxen blinxen Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Copilot This seems to be an issue with .editorconfig. I changed it in my latest commit.

s.Require().NoError(err)

foundRuleIds := ll.TriggeredRules()
s.Len(foundRuleIds, 1)
s.Contains(foundRuleIds, uint(920300))
}
10 changes: 9 additions & 1 deletion waflog/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package waflog

import (
"os"

"regexp"
"slices"

"github.com/coreruleset/go-ftw/v2/config"
Expand All @@ -24,6 +24,8 @@ type FTWLogLines struct {
markedLinesInitialized bool
triggeredRulesInitialized bool
runMode config.RunMode
stdLogIdRegex *regexp.Regexp
jsonLogIdRegex *regexp.Regexp
}

func (ll *FTWLogLines) StartMarker() []byte {
Expand All @@ -41,4 +43,10 @@ func (ll *FTWLogLines) reset() {
ll.markedLines = slices.Delete(ll.markedLines, 0, len(ll.markedLines))
ll.markedLinesInitialized = false
ll.triggeredRulesInitialized = false
if ll.stdLogIdRegex == nil {
ll.stdLogIdRegex = regexp.MustCompile(config.DefaultStdLogIdRegex)
}
if ll.jsonLogIdRegex == nil {
ll.jsonLogIdRegex = regexp.MustCompile(config.DefaultJsonLogIdRegex)
}
}
20 changes: 20 additions & 0 deletions waflog/waflog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,27 @@ import (
"errors"
"fmt"
"os"
"regexp"

"github.com/coreruleset/go-ftw/v2/config"
)

// NewFTWLogLines is the base struct for reading the log file
func NewFTWLogLines(cfg *config.RunnerConfig) (*FTWLogLines, error) {
stdLogIdRegex, err := regexp.Compile(cfg.StdLogIdRegex)
if err != nil {
return nil, fmt.Errorf("could not compile stdLogIdRegex: %s", err)
}
jsonLogIdRegex, err := regexp.Compile(cfg.JsonLogIdRegex)
if err != nil {
return nil, fmt.Errorf("could not compile jsonLogIdRegex: %s", err)
Comment thread
blinxen marked this conversation as resolved.
Outdated
}
ll := &FTWLogLines{
logFilePath: cfg.LogFilePath,
runMode: cfg.RunMode,
LogMarkerHeaderName: bytes.ToLower([]byte(cfg.LogMarkerHeaderName)),
stdLogIdRegex: stdLogIdRegex,
jsonLogIdRegex: jsonLogIdRegex,
}
Comment thread
blinxen marked this conversation as resolved.

if err := ll.openLogFile(); err != nil {
Expand All @@ -42,6 +53,15 @@ func (ll *FTWLogLines) WithEndMarker(marker []byte) {
ll.endMarker = bytes.ToLower(marker)
}

func (ll *FTWLogLines) WithStdLogIdRegex(regex string) error {
compiledRegex, err := regexp.Compile(regex)
if err != nil {
return err
}
ll.stdLogIdRegex = compiledRegex
return nil
}

Copy link

Copilot AI Apr 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WithStdLogIdRegex introduces a public override hook for only the std log regex, but there’s no equivalent for the JSON log regex. Either add a matching WithJsonLogIdRegex (and consider a combined setter), or keep these overrides internal to tests via config to avoid an asymmetric public API.

Suggested change
func (ll *FTWLogLines) WithJsonLogIdRegex(regex string) error {
compiledRegex, err := regexp.Compile(regex)
if err != nil {
return err
}
ll.jsonLogIdRegex = compiledRegex
return nil
}

Copilot uses AI. Check for mistakes.
Copy link
Copy Markdown
Author

@blinxen blinxen Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Copilot

What do you mean with

keep these overrides internal to tests via config

// Cleanup closes the log file
func (ll *FTWLogLines) Cleanup() error {
if ll != nil && ll.logFile != nil {
Expand Down
3 changes: 3 additions & 0 deletions waflog/waflog_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package waflog

import (
"os"
"regexp"
"testing"

"github.com/rs/zerolog"
Expand Down Expand Up @@ -77,4 +78,6 @@ func (s *waflogTestSuite) TestLogLinesReset() {
s.Nil(ll.endMarker)
s.Empty(ll.triggeredRules)
s.Empty(ll.markedLines)
s.Equal(regexp.MustCompile(config.DefaultStdLogIdRegex).String(), ll.stdLogIdRegex.String())
s.Equal(regexp.MustCompile(config.DefaultJsonLogIdRegex).String(), ll.jsonLogIdRegex.String())
}