Skip to content

Commit 6a6f1bc

Browse files
caicancaicaicancai
committed
log/logstorage/ipv6: Support ipv6_range for VictoriaLogs
Signed-off-by: cancaicai <[email protected]>
1 parent e300c83 commit 6a6f1bc

File tree

4 files changed

+458
-0
lines changed

4 files changed

+458
-0
lines changed

‎lib/logstorage/filter_ipv6_range.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package logstorage
2+
3+
import (
4+
"fmt"
5+
"net"
6+
7+
"github.com/VictoriaMetrics/VictoriaLogs/lib/prefixfilter"
8+
)
9+
10+
// filterIPv6Range matches the given ipv6 range [minValue..maxValue].
11+
//
12+
// Example LogsQL: `fieldName:ipv6_range(::1, ::2)`
13+
type filterIPv6Range struct {
14+
fieldName string
15+
minValue string
16+
maxValue string
17+
}
18+
19+
func (fr *filterIPv6Range) String() string {
20+
minIP := net.IP([]byte(fr.minValue))
21+
maxIP := net.IP([]byte(fr.maxValue))
22+
return fmt.Sprintf("%sipv6_range(%s, %s)", quoteFieldNameIfNeeded(fr.fieldName), minIP.String(), maxIP.String())
23+
}
24+
25+
func (fr *filterIPv6Range) updateNeededFields(pf *prefixfilter.Filter) {
26+
pf.AddAllowFilter(fr.fieldName)
27+
}
28+
29+
func (fr *filterIPv6Range) matchRow(fields []Field) bool {
30+
v := getFieldValueByName(fields, fr.fieldName)
31+
return matchIPv6Range(v, fr.minValue, fr.maxValue)
32+
}
33+
34+
func (fr *filterIPv6Range) applyToBlockResult(br *blockResult, bm *bitmap) {
35+
minValue := fr.minValue
36+
maxValue := fr.maxValue
37+
38+
if minValue > maxValue {
39+
bm.resetBits()
40+
return
41+
}
42+
43+
c := br.getColumnByName(fr.fieldName)
44+
if c.isConst {
45+
v := c.valuesEncoded[0]
46+
if !matchIPv6Range(v, minValue, maxValue) {
47+
bm.resetBits()
48+
}
49+
return
50+
}
51+
if c.isTime {
52+
bm.resetBits()
53+
return
54+
}
55+
56+
switch c.valueType {
57+
case valueTypeString:
58+
values := c.getValues(br)
59+
bm.forEachSetBit(func(idx int) bool {
60+
v := values[idx]
61+
return matchIPv6Range(v, minValue, maxValue)
62+
})
63+
case valueTypeDict:
64+
bb := bbPool.Get()
65+
for _, v := range c.dictValues {
66+
c := byte(0)
67+
if matchIPv6Range(v, minValue, maxValue) {
68+
c = 1
69+
}
70+
bb.B = append(bb.B, c)
71+
}
72+
valuesEncoded := c.getValuesEncoded(br)
73+
bm.forEachSetBit(func(idx int) bool {
74+
n := valuesEncoded[idx][0]
75+
return bb.B[n] == 1
76+
})
77+
bbPool.Put(bb)
78+
default:
79+
bm.resetBits()
80+
}
81+
}
82+
83+
func (fr *filterIPv6Range) applyToBlockSearch(bs *blockSearch, bm *bitmap) {
84+
fieldName := fr.fieldName
85+
minValue := fr.minValue
86+
maxValue := fr.maxValue
87+
88+
if minValue > maxValue {
89+
bm.resetBits()
90+
return
91+
}
92+
93+
v := bs.getConstColumnValue(fieldName)
94+
if v != "" {
95+
if !matchIPv6Range(v, minValue, maxValue) {
96+
bm.resetBits()
97+
}
98+
return
99+
}
100+
101+
// Verify whether filter matches other columns
102+
ch := bs.getColumnHeader(fieldName)
103+
if ch == nil {
104+
// Fast path - there are no matching columns.
105+
bm.resetBits()
106+
return
107+
}
108+
109+
switch ch.valueType {
110+
case valueTypeString:
111+
matchStringByIPv6Range(bs, ch, bm, minValue, maxValue)
112+
case valueTypeDict:
113+
matchValuesDictByIPv6Range(bs, ch, bm, minValue, maxValue)
114+
default:
115+
bm.resetBits()
116+
}
117+
}
118+
119+
func matchValuesDictByIPv6Range(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
120+
bb := bbPool.Get()
121+
for _, v := range ch.valuesDict.values {
122+
c := byte(0)
123+
if matchIPv6Range(v, minValue, maxValue) {
124+
c = 1
125+
}
126+
bb.B = append(bb.B, c)
127+
}
128+
matchEncodedValuesDict(bs, ch, bm, bb.B)
129+
bbPool.Put(bb)
130+
}
131+
132+
func matchStringByIPv6Range(bs *blockSearch, ch *columnHeader, bm *bitmap, minValue, maxValue string) {
133+
visitValues(bs, ch, bm, func(v string) bool {
134+
return matchIPv6Range(v, minValue, maxValue)
135+
})
136+
}
137+
138+
func matchIPv6Range(s string, minValue, maxValue string) bool {
139+
ip, ok := tryParseIPv6(s)
140+
if !ok {
141+
return false
142+
}
143+
return ip >= minValue && ip <= maxValue
144+
}

‎lib/logstorage/filter_ipv6_range_test.go‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package logstorage
2+
3+
import (
4+
"net"
5+
"testing"
6+
7+
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
8+
)
9+
10+
func TestMatchIPv6Range(t *testing.T) {
11+
t.Parallel()
12+
13+
f := func(s, minValue, maxValue string, resultExpected bool) {
14+
t.Helper()
15+
minIP := string(net.ParseIP(minValue).To16())
16+
maxIP := string(net.ParseIP(maxValue).To16())
17+
result := matchIPv6Range(s, minIP, maxIP)
18+
if result != resultExpected {
19+
t.Fatalf("unexpected result; got %v; want %v", result, resultExpected)
20+
}
21+
}
22+
23+
// Invalid IP
24+
f("", "::1", "::2", false)
25+
f("123", "::1", "::2", false)
26+
f("1.2.3.4", "::1", "::2", false)
27+
28+
// range mismatch
29+
f("::1", "::2", "::3", false)
30+
f("2001:db8::1", "2001:db8::2", "2001:db8::3", false)
31+
32+
// range match
33+
f("::1", "::1", "::1", true)
34+
f("::1", "::0", "::2", true)
35+
f("2001:db8::1", "2001:db8::", "2001:db8::ffff", true)
36+
}
37+
38+
func TestFilterIPv6Range(t *testing.T) {
39+
t.Parallel()
40+
41+
parseIP := func(s string) string {
42+
return string(net.ParseIP(s).To16())
43+
}
44+
45+
t.Run("const-column", func(t *testing.T) {
46+
columns := []column{
47+
{
48+
name: "foo",
49+
values: []string{
50+
"::1",
51+
"::1",
52+
"::1",
53+
},
54+
},
55+
}
56+
57+
// match
58+
fr := &filterIPv6Range{
59+
fieldName: "foo",
60+
minValue: parseIP("::0"),
61+
maxValue: parseIP("::2"),
62+
}
63+
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
64+
65+
fr = &filterIPv6Range{
66+
fieldName: "foo",
67+
minValue: parseIP("::1"),
68+
maxValue: parseIP("::1"),
69+
}
70+
testFilterMatchForColumns(t, columns, fr, "foo", []int{0, 1, 2})
71+
72+
// mismatch
73+
fr = &filterIPv6Range{
74+
fieldName: "foo",
75+
minValue: parseIP("::2"),
76+
maxValue: parseIP("::3"),
77+
}
78+
testFilterMatchForColumns(t, columns, fr, "foo", nil)
79+
80+
fr = &filterIPv6Range{
81+
fieldName: "non-existing-column",
82+
minValue: parseIP("::0"),
83+
maxValue: parseIP("::ffff"),
84+
}
85+
testFilterMatchForColumns(t, columns, fr, "foo", nil)
86+
87+
fr = &filterIPv6Range{
88+
fieldName: "foo",
89+
minValue: parseIP("::2"),
90+
maxValue: parseIP("::0"),
91+
}
92+
testFilterMatchForColumns(t, columns, fr, "foo", nil)
93+
})
94+
95+
t.Run("dict", func(t *testing.T) {
96+
columns := []column{
97+
{
98+
name: "foo",
99+
values: []string{
100+
"",
101+
"::1",
102+
"Abc",
103+
"2001:db8::1",
104+
"10.4",
105+
"foo ::1",
106+
"::1 bar",
107+
"::1",
108+
},
109+
},
110+
}
111+
112+
// match
113+
fr := &filterIPv6Range{
114+
fieldName: "foo",
115+
minValue: parseIP("::0"),
116+
maxValue: parseIP("::2"),
117+
}
118+
testFilterMatchForColumns(t, columns, fr, "foo", []int{1, 7})
119+
120+
fr = &filterIPv6Range{
121+
fieldName: "foo",
122+
minValue: parseIP("2001:db8::"),
123+
maxValue: parseIP("2001:db8::ffff"),
124+
}
125+
testFilterMatchForColumns(t, columns, fr, "foo", []int{3})
126+
127+
// mismatch
128+
fr = &filterIPv6Range{
129+
fieldName: "foo",
130+
minValue: parseIP("::3"),
131+
maxValue: parseIP("::4"),
132+
}
133+
testFilterMatchForColumns(t, columns, fr, "foo", nil)
134+
})
135+
136+
t.Run("strings", func(t *testing.T) {
137+
columns := []column{
138+
{
139+
name: "foo",
140+
values: []string{
141+
"A FOO",
142+
"a 10",
143+
"::1",
144+
"20",
145+
"15.5",
146+
"-5",
147+
"a fooBaR",
148+
"a ::1 dfff",
149+
"a ТЕСТЙЦУК НГКШ ",
150+
"a !!,23.(!1)",
151+
"2001:db8::1",
152+
},
153+
},
154+
}
155+
156+
// match
157+
fr := &filterIPv6Range{
158+
fieldName: "foo",
159+
minValue: parseIP("::0"),
160+
maxValue: parseIP("::2"),
161+
}
162+
testFilterMatchForColumns(t, columns, fr, "foo", []int{2})
163+
164+
fr = &filterIPv6Range{
165+
fieldName: "foo",
166+
minValue: parseIP("2001:db8::"),
167+
maxValue: parseIP("2001:db8::ffff"),
168+
}
169+
testFilterMatchForColumns(t, columns, fr, "foo", []int{10})
170+
171+
// mismatch
172+
fr = &filterIPv6Range{
173+
fieldName: "foo",
174+
minValue: parseIP("::3"),
175+
maxValue: parseIP("::4"),
176+
}
177+
testFilterMatchForColumns(t, columns, fr, "foo", nil)
178+
})
179+
180+
// Remove the remaining data files for the test
181+
fs.MustRemoveDir(t.Name())
182+
}

0 commit comments

Comments
 (0)