Skip to content

Commit b5ec14f

Browse files
RomainMullerRomainMuller
authored
feat: embed the default ruleset in go-libddwaf (#129)
1 parent 1c4cdc9 commit b5ec14f

File tree

11 files changed

+505
-7
lines changed

11 files changed

+505
-7
lines changed

‎.github/workflows/default-ruleset.yml‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
name: Default Ruleset
2+
on:
3+
workflow_dispatch: # Manually
4+
schedule: # Every Monday at 06:00 UTC
5+
- cron: '0 6 * * 1'
6+
7+
permissions: read-all
8+
9+
10+
jobs:
11+
update:
12+
runs-on: ubuntu-latest
13+
name: Update
14+
outputs:
15+
mutation-happened: ${{ steps.detect.outputs.mutation-happened }}
16+
steps:
17+
- name: Checkout
18+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
19+
- name: Setup Go
20+
uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0
21+
with:
22+
go-version: oldstable
23+
cache-dependency-path: _tools/ruleset-updater/go.mod
24+
- name: Generate a GitHub token
25+
id: generate-token
26+
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
27+
with:
28+
app-id: ${{ vars.DD_K9_LIBRARY_GO_APP_ID }}
29+
private-key: ${{ secrets.DD_K9_LIBRARY_GO_APP_PRIVATE_KEY }}
30+
- name: Update Default Ruleset
31+
run: go run -C _tools/ruleset-updater run . -output=${{ github.workspace }}/internal/ruleset/recommended.json.gz
32+
env:
33+
GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
34+
- name: Detect Mutation
35+
id: detect
36+
run: |-
37+
git add .
38+
git diff --staged --patch --exit-code > ${{ runner.temp }}/repo.patch || echo "mutation-happened=true" >> "${GITHUB_OUTPUT}"
39+
- name: Upload Patch
40+
if: steps.detect.outputs.mutation_happened
41+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
42+
with:
43+
name: repo.patch
44+
path: ${{ runner.temp }}/repo.patch
45+
46+
pr:
47+
runs-on: ubuntu-latest
48+
name: Create PR
49+
needs: update
50+
if: needs.update.outputs.mutation-happened
51+
permissions:
52+
contents: write
53+
steps:
54+
- name: Checkout
55+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
56+
- name: Download Patch
57+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
58+
with:
59+
name: repo.patch
60+
path: ${{ runner.temp }}
61+
- name: Apply Patch
62+
id: apply
63+
run: |-
64+
git apply ${{ runner.temp }}/repo.patch
65+
echo "version=$(jq -r '.metadata.rules_version' < ./appsec/rules.json)" >> $GITHUB_OUTPUT
66+
67+
- name: Create PR Branch
68+
id: create-branch
69+
run: |-
70+
branch="automation/default-ruleset-update/${VERSION}"
71+
git push origin "${{ github.sha }}":"refs/heads/${branch}"
72+
echo "branch=${branch}" >> "${GITHUB_OUTPUT}"
73+
git fetch origin "${branch}"
74+
env:
75+
VERSION: ${{ steps.apply.outputs.version }}
76+
- name: Generate a GitHub token
77+
id: generate-token
78+
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
79+
with:
80+
app-id: ${{ vars.DD_K9_LIBRARY_GO_APP_ID }}
81+
private-key: ${{ secrets.DD_K9_LIBRARY_GO_APP_PRIVATE_KEY }}
82+
# We use ghcommit to create signed commits directly using the GitHub API
83+
- name: Create Commit on PR Branch
84+
uses: planetscale/ghcommit-action@6a383e778f6620afde4bf4b45069d3c6983c1ae2 # v0.2.15
85+
with:
86+
commit_message: "chore: update default ruleset to ${{ steps.apply.outputs.version }}"
87+
branch: ${{ steps.create-branch.outputs.branch }}
88+
repo: ${{ github.repository }}
89+
env:
90+
GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
91+
- name: Create PR
92+
run: |-
93+
git fetch origin "${{ steps.create-branch.outputs.branch }}"
94+
git reset --hard HEAD
95+
git switch "${{ steps.create-branch.outputs.branch }}"
96+
gh pr create --title "chore: update default ruleset to ${VERSION}" \
97+
--body "Updated default ruleset to ${VERSION}." \
98+
--head="${{ steps.create-branch.outputs.branch }}"
99+
env:
100+
GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }}
101+
VERSION: ${{ steps.apply.outputs.version }}

‎_tools/ruleset-updater/go.mod‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
module github.com/DataDog/go-libddwaf/_tools/ruleset-updater
2+
3+
go 1.23.0
4+
5+
toolchain go1.24.3
6+
7+
require (
8+
github.com/google/go-github/v72 v72.0.0
9+
github.com/iancoleman/orderedmap v0.3.0
10+
)
11+
12+
require github.com/google/go-querystring v1.1.0 // indirect

‎_tools/ruleset-updater/go.sum‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
2+
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
3+
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
4+
github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM=
5+
github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg=
6+
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
7+
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
8+
github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc=
9+
github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE=
10+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

‎_tools/ruleset-updater/main.go‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2016-present Datadog, Inc.
5+
6+
package main
7+
8+
import (
9+
"compress/gzip"
10+
"context"
11+
"encoding/json"
12+
"flag"
13+
"log"
14+
"os"
15+
16+
"github.com/google/go-github/v72/github"
17+
"github.com/iancoleman/orderedmap"
18+
)
19+
20+
const (
21+
ghRepoOwner = "DataDog"
22+
ghRepoName = "appsec-event-rules"
23+
)
24+
25+
func main() {
26+
var output string
27+
flag.StringVar(&output, "output", "", "Path to the output file")
28+
flag.Parse()
29+
30+
if output == "" {
31+
log.Fatalln("Missing required flag: -output")
32+
}
33+
34+
token := os.Getenv("GITHUB_TOKEN")
35+
if token == "" {
36+
log.Fatalln("GITHUB_TOKEN is not set; this is required to update the default ruleset!")
37+
}
38+
39+
gh := github.NewClient(nil).WithAuthToken(token)
40+
41+
ctx := context.Background()
42+
43+
release, _, err := gh.Repositories.GetLatestRelease(ctx, ghRepoOwner, ghRepoName)
44+
if err != nil {
45+
log.Fatalln("Failed to get latest release:", err)
46+
}
47+
48+
tag := release.GetTagName()
49+
log.Println("Latest release is", tag)
50+
51+
file, _, err := gh.Repositories.DownloadContents(ctx, ghRepoOwner, ghRepoName, "build/recommended.json", &github.RepositoryContentGetOptions{Ref: tag})
52+
if err != nil {
53+
log.Fatalln("Failed to get recommended.json:", err)
54+
}
55+
defer file.Close()
56+
dec := json.NewDecoder(file)
57+
dec.UseNumber()
58+
59+
// Decode the original ruleset into an [orderedmap.OrderedMap] so we preserve the order of items
60+
// from the original file. This makes it easier to ensure the output GZIP file is always the same
61+
// given a version of the input file.
62+
var ruleset orderedmap.OrderedMap
63+
if err := dec.Decode(&ruleset); err != nil {
64+
log.Fatalln("Failed to decode recommended.json:", err)
65+
}
66+
67+
out, err := os.Create(output)
68+
if err != nil {
69+
log.Fatalln("Failed to create output file:", err)
70+
}
71+
defer out.Close()
72+
73+
wr, err := gzip.NewWriterLevel(out, gzip.BestCompression)
74+
if err != nil {
75+
log.Fatalln("Failed to create gzip writer:", err)
76+
}
77+
defer wr.Close()
78+
79+
// Strip all irrelevant whitespace from the JSON so it is as small as possible.
80+
enc := json.NewEncoder(wr)
81+
enc.SetIndent("", "")
82+
83+
if err := enc.Encode(ruleset); err != nil {
84+
log.Fatalln("Failed to encode ruleset:", err)
85+
}
86+
}

‎internal/bindings/ctypes.go‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -145,11 +145,21 @@ func (w *WAFObject) SetArray(pinner pin.Pinner, capacity uint64) []WAFObject {
145145
return w.setArrayTyped(pinner, capacity, WAFArrayType)
146146
}
147147

148+
// SetArrayData sets the receiving [WAFObject] to the provided array items.
149+
func (w *WAFObject) SetArrayData(pinner pin.Pinner, data []WAFObject) {
150+
w.setArrayDataTyped(pinner, data, WAFArrayType)
151+
}
152+
148153
// SetMap sets the receiving [WAFObject] to a new map with the given capacity.
149154
func (w *WAFObject) SetMap(pinner pin.Pinner, capacity uint64) []WAFObject {
150155
return w.setArrayTyped(pinner, capacity, WAFMapType)
151156
}
152157

158+
// SetMapData sets the receiving [WAFObject] to the provided map items.
159+
func (w *WAFObject) SetMapData(pinner pin.Pinner, data []WAFObject) {
160+
w.setArrayDataTyped(pinner, data, WAFMapType)
161+
}
162+
153163
// SetMapKey sets the receiving [WAFObject] to a new map key with the given
154164
// string.
155165
func (w *WAFObject) SetMapKey(pinner pin.Pinner, key string) {
@@ -339,18 +349,25 @@ func (w *WAFObject) SetInvalid() {
339349
}
340350

341351
func (w *WAFObject) setArrayTyped(pinner pin.Pinner, capacity uint64, t WAFObjectType) []WAFObject {
352+
var arr []WAFObject
353+
if capacity > 0 {
354+
arr = make([]WAFObject, capacity)
355+
}
356+
w.setArrayDataTyped(pinner, arr, t)
357+
return arr
358+
}
359+
360+
func (w *WAFObject) setArrayDataTyped(pinner pin.Pinner, arr []WAFObject, t WAFObjectType) {
342361
w.Type = t
343-
w.NbEntries = capacity
362+
w.NbEntries = uint64(len(arr))
344363
if w.NbEntries == 0 {
345364
w.Value = 0
346-
return nil
365+
return
347366
}
348367

349-
arr := make([]WAFObject, capacity)
350-
data := unsafe.Pointer(unsafe.SliceData(arr))
351-
pinner.Pin(data)
352-
w.Value = uintptr(data)
353-
return arr
368+
ptr := unsafe.Pointer(unsafe.SliceData(arr))
369+
pinner.Pin(ptr)
370+
w.Value = uintptr(ptr)
354371
}
355372

356373
type WAFConfig struct {

‎internal/ruleset/.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/recommended.json.gz linguist-vendored

‎internal/ruleset/recommended.json.gz‎

29.7 KB
Binary file not shown.

‎internal/ruleset/ruleset.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2016-present Datadog, Inc.
5+
6+
package ruleset
7+
8+
import (
9+
"bytes"
10+
"compress/gzip"
11+
_ "embed"
12+
"runtime"
13+
14+
"github.com/DataDog/go-libddwaf/v4/internal/bindings"
15+
"github.com/DataDog/go-libddwaf/v4/json"
16+
) // For go:embed
17+
18+
//go:embed recommended.json.gz
19+
var defaultRuleset []byte
20+
21+
func DefaultRuleset(pinner *runtime.Pinner) (bindings.WAFObject, error) {
22+
gz, err := gzip.NewReader(bytes.NewReader(defaultRuleset))
23+
if err != nil {
24+
return bindings.WAFObject{}, err
25+
}
26+
27+
dec := json.NewDecoder(gz, pinner)
28+
29+
var ruleset bindings.WAFObject
30+
if err := dec.Decode(&ruleset); err != nil {
31+
return bindings.WAFObject{}, err
32+
}
33+
return ruleset, nil
34+
}

‎internal/ruleset/ruleset_test.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed
2+
// under the Apache License Version 2.0.
3+
// This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
// Copyright 2016-present Datadog, Inc.
5+
6+
package ruleset
7+
8+
import (
9+
"bytes"
10+
"compress/gzip"
11+
"encoding/json"
12+
"runtime"
13+
"strconv"
14+
"testing"
15+
16+
"github.com/stretchr/testify/require"
17+
)
18+
19+
func TestDefaultRuleset(t *testing.T) {
20+
var pinner runtime.Pinner
21+
defer pinner.Unpin()
22+
23+
rs, err := DefaultRuleset(&pinner)
24+
require.NoError(t, err)
25+
require.NotNil(t, rs)
26+
27+
obj, err := rs.MapValue()
28+
require.NoError(t, err)
29+
useJSONNumber(obj) // So we can easily compare the result with the expected JSON.
30+
31+
rd, err := gzip.NewReader(bytes.NewReader(defaultRuleset))
32+
require.NoError(t, err)
33+
dec := json.NewDecoder(rd)
34+
dec.UseNumber()
35+
var expected map[string]any
36+
require.NoError(t, dec.Decode(&expected))
37+
require.Equal(t, expected, obj)
38+
}
39+
40+
func useJSONNumber(obj map[string]any) {
41+
for k, v := range obj {
42+
switch v := v.(type) {
43+
case float64:
44+
obj[k] = json.Number(strconv.FormatFloat(v, 'f', -1, 64))
45+
case int64:
46+
obj[k] = json.Number(strconv.FormatInt(v, 10))
47+
case uint64:
48+
obj[k] = json.Number(strconv.FormatUint(v, 10))
49+
case map[string]any:
50+
useJSONNumber(v)
51+
case []any:
52+
for _, v := range v {
53+
if v, ok := v.(map[string]any); ok {
54+
useJSONNumber(v)
55+
}
56+
}
57+
}
58+
}
59+
}

0 commit comments

Comments
 (0)