Skip to content
This repository was archived by the owner on Nov 7, 2025. It is now read-only.

Commit c54fcc2

Browse files
trzysiektrzysiek
authored
ip_range: support for ipv6 (#1157)
1 parent e52d81d commit c54fcc2

File tree

7 files changed

+170
-28
lines changed

7 files changed

+170
-28
lines changed

‎docs/public/docs/limitations.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ Currently supported:
3434
including: `boolean`, `match`, `match phrase`, `multi-match`, `query string`, `nested`, `match all`, `exists`, `prefix`, `range`, `term`, `terms`, `wildcard`
3535
- most popular [Aggregations](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html),
3636
including: `avg`, `cardinality`, `max`, `min`, `percentile ranks`, `percentiles`, `stats`, `sum`, `top hits`, `top metrics`, `value counts`,
37-
`date histogram`, `date range`, `filter`, `filters`, `histogram`, `range`, `singificant terms`, `terms`, `ip prefix`
37+
`date histogram`, `date range`, `filter`, `filters`, `histogram`, `range`, `singificant terms`, `terms`, `ip prefix`, `ip range`
3838

3939
Which as a result allows you to run Kibana/OSD queries and dashboards on data residing in ClickHouse/Hydrolix.
4040

‎quesma/go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ require (
4141
require (
4242
filippo.io/edwards25519 v1.1.0 // indirect
4343
github.com/H0llyW00dzZ/cidr v1.2.1 // indirect
44+
github.com/apparentlymart/go-cidr v1.1.0 // indirect
4445
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
4546
github.com/hashicorp/errwrap v1.0.0 // indirect
4647
github.com/jackc/chunkreader/v2 v2.0.1 // indirect

‎quesma/go.sum‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,17 @@ github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h
77
github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo=
88
github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=
99
github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=
10-
github.com/H0llyW00dzZ/cidr v1.2.1 h1:DfRHX+RqVVKZijQGO1aJSaWvN9Saan8sycK/4wrfY5g=
11-
github.com/H0llyW00dzZ/cidr v1.2.1/go.mod h1:S+EgYkMandSAN27mGNG/CB3jeoXDAyalsvvVFpWdnXc=
1210
github.com/DataDog/go-sqllexer v0.0.18 h1:ErBvoO7/srJLdA2ebwd+HPqD4g1kN++BP64A8qvmh9U=
1311
github.com/DataDog/go-sqllexer v0.0.18/go.mod h1:KwkYhpFEVIq+BfobkTC1vfqm4gTi65skV/DpDBXtexc=
12+
github.com/H0llyW00dzZ/cidr v1.2.1 h1:DfRHX+RqVVKZijQGO1aJSaWvN9Saan8sycK/4wrfY5g=
13+
github.com/H0llyW00dzZ/cidr v1.2.1/go.mod h1:S+EgYkMandSAN27mGNG/CB3jeoXDAyalsvvVFpWdnXc=
1414
github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
1515
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
1616
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
1717
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
1818
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
19+
github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU=
20+
github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc=
1921
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
2022
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
2123
github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=

‎quesma/model/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ More info: https://www.elastic.co/guide/en/elasticsearch/reference/current/searc
2828
Median absolute deviation | :x: | Global | :x: | Moving function | :wavy_dash: |
2929
Min | :white_check_mark: | Histogram | :white_check_mark: | Moving percentiles | :x: |
3030
Percentile ranks | :white_check_mark: | IP prefix | :white_check_mark: | Normalize | :x: |
31-
Percentiles | :white_check_mark: | IP range | :x: | Percentiles bucket | :x: |
31+
Percentiles | :white_check_mark: | IP range | :white_check_mark: | Percentiles bucket | :x: |
3232
Rate | :x: | Missing | :x: | Serial differencing | :white_check_mark: |
3333
Scripted metric | :x: | Multi-terms | :white_check_mark: | Stats bucket | :x: |
3434
Stats | :white_check_mark: | Nested | :x: | Sum bucket | :white_check_mark: |

‎quesma/model/bucket_aggregations/ip_range.go‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package bucket_aggregations
55
import (
66
"context"
77
"fmt"
8+
"net/netip"
89
"quesma/logger"
910
"quesma/model"
1011
"reflect"
@@ -14,8 +15,6 @@ import (
1415
// So instead of "<= 255.255.255.255", it uses "< ::1:0:0:0"
1516
const BiggestIpv4 = "::1:0:0:0"
1617

17-
// Current limitation: we expect Clickhouse field to be IPv4 (and not IPv6)
18-
1918
// Clickhouse table to test SQLs:
2019
// CREATE TABLE __quesma_table_name (clientip IPv4) ENGINE=Log
2120
// INSERT INTO __quesma_table_name VALUES ('0.0.0.0'), ('5.5.5.5'), ('90.180.90.180'), ('128.200.0.8'), ('192.168.1.67'), ('222.168.22.67')
@@ -95,23 +94,34 @@ func NewIpInterval(begin, end string, key *string) IpInterval {
9594
}
9695

9796
func (interval IpInterval) ToWhereClause(field model.Expr) model.Expr {
98-
isBegin := interval.begin != UnboundedInterval
99-
isEnd := interval.end != UnboundedInterval && interval.end != BiggestIpv4
97+
hasBegin := interval.hasBeginInResponse()
98+
hasEnd := interval.hasEndInResponse()
10099

101100
begin := model.NewInfixExpr(field, ">=", model.NewLiteralSingleQuoteString(interval.begin))
102101
end := model.NewInfixExpr(field, "<", model.NewLiteralSingleQuoteString(interval.end))
103102

104-
if isBegin && isEnd {
103+
if hasBegin && hasEnd {
105104
return model.NewInfixExpr(begin, "AND", end)
106-
} else if isBegin {
105+
} else if hasBegin {
107106
return begin
108-
} else if isEnd {
107+
} else if hasEnd {
109108
return end
110109
} else {
111110
return model.TrueExpr
112111
}
113112
}
114113

114+
// hasBeginInResponse returns true if we should add 'from' field to the response.
115+
// We do that <=> begin is not 0.0.0.0 (unbounded)
116+
func (interval IpInterval) hasBeginInResponse() bool {
117+
return interval.begin != UnboundedInterval && netip.MustParseAddr(interval.begin) != netip.MustParseAddr("::")
118+
}
119+
120+
// hasEndInResponse returns true if we should add 'to' field to the response.
121+
func (interval IpInterval) hasEndInResponse() bool {
122+
return interval.end != UnboundedInterval
123+
}
124+
115125
// String returns key part of the response, e.g. "1.0-2.0", or "*-6.55"
116126
func (interval IpInterval) String() string {
117127
if interval.key != nil {
@@ -166,10 +176,10 @@ func (query *IpRange) CombinatorTranslateSqlResponseToJson(subGroup CombinatorGr
166176
}
167177

168178
interval := query.intervals[subGroup.idx]
169-
if interval.begin != UnboundedInterval {
179+
if interval.hasBeginInResponse() {
170180
response["from"] = interval.begin
171181
}
172-
if interval.end != UnboundedInterval {
182+
if interval.hasEndInResponse() {
173183
response["to"] = interval.end
174184
}
175185

‎quesma/queryparser/pancake_aggregation_parser_buckets.go‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ package queryparser
66
import (
77
"fmt"
88
"github.com/H0llyW00dzZ/cidr"
9+
cidr2 "github.com/apparentlymart/go-cidr/cidr"
910
"github.com/pkg/errors"
1011
"math"
1112
"net"
13+
"net/netip"
1214
"quesma/clickhouse"
1315
"quesma/logger"
1416
"quesma/model"
@@ -410,23 +412,39 @@ func (cw *ClickhouseQueryTranslator) parseIpRange(aggregation *pancakeAggregatio
410412
rangesRaw := params["ranges"].([]any)
411413
ranges := make([]bucket_aggregations.IpInterval, 0, len(rangesRaw))
412414
for _, rangeRaw := range rangesRaw {
415+
var begin, end string
413416
var key *string
414417
if keyIfPresent, exists := cw.parseStringFieldExistCheck(rangeRaw.(QueryMap), "key"); exists {
415418
key = &keyIfPresent
416419
}
417-
var begin, end string
418420
if maskIfExists, exists := cw.parseStringFieldExistCheck(rangeRaw.(QueryMap), "mask"); exists {
419421
_, ipNet, err := net.ParseCIDR(maskIfExists)
420422
if err != nil {
421423
return err
422424
}
423-
beginAsInt, endAsInt := cidr.IPv4ToRange(ipNet)
424-
begin = util.IntToIpv4(beginAsInt)
425-
// endAsInt is inclusive, we do +1, because we need it exclusive
426-
if endAsInt != math.MaxUint32 {
427-
end = util.IntToIpv4(endAsInt + 1)
425+
if ipNet.IP.To4() != nil {
426+
// it's ipv4
427+
beginAsInt, endAsInt := cidr.IPv4ToRange(ipNet)
428+
begin = util.IntToIpv4(beginAsInt)
429+
// endAsInt is inclusive, we do +1, because we need it exclusive
430+
if endAsInt != math.MaxUint32 {
431+
end = util.IntToIpv4(endAsInt + 1)
432+
} else {
433+
end = bucket_aggregations.BiggestIpv4 // "255.255.255.255 + 1", so to say (value in compliance with Elastic)
434+
}
435+
} else if ipNet.IP.To16() != nil {
436+
// it's ipv6
437+
beginInclusive, endInclusive := cidr2.AddressRange(ipNet)
438+
begin = beginInclusive.String()
439+
// we do +1 (.Next()), because we need end to be exclusive
440+
endExclusive := netip.MustParseAddr(endInclusive.String()).Next()
441+
if endExclusive.IsValid() {
442+
end = endExclusive.String()
443+
} else { // invalid means endInclusive was already the biggest possible value (ff...ff)
444+
end = bucket_aggregations.UnboundedInterval
445+
}
428446
} else {
429-
end = bucket_aggregations.BiggestIpv4 // "255.255.255.255 + 1", so to say (value in compliance with Elastic)
447+
return fmt.Errorf("invalid mask: %s", maskIfExists)
430448
}
431449
if key == nil {
432450
key = &maskIfExists

‎quesma/testdata/kibana-visualize/aggregation_requests.go‎

Lines changed: 119 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3375,9 +3375,6 @@ var AggregationTests = []testdata.AggregationTestCase{
33753375
TestName: "IP range, with ranges as CIDR masks. In Kibana: Add panel > Aggregation Based > Area. Buckets: X-asis: IP Range",
33763376
QueryRequestJson: `
33773377
{
3378-
"_source": {
3379-
"excludes": []
3380-
},
33813378
"aggs": {
33823379
"2": {
33833380
"ip_range": {
@@ -3451,7 +3448,7 @@ var AggregationTests = []testdata.AggregationTestCase{
34513448
}},
34523449
},
34533450
ExpectedPancakeSQL: `
3454-
SELECT countIf("clientip">='255.255.255.252') AS "range_0__aggr__2__count",
3451+
SELECT countIf(("clientip">='255.255.255.252' AND "clientip"<'::1:0:0:0')) AS "range_0__aggr__2__count",
34553452
countIf("clientip">='128.129.130.131') AS "range_1__aggr__2__count",
34563453
countIf(("clientip">='10.0.7.96' AND "clientip"<'10.0.7.128')) AS
34573454
"range_2__aggr__2__count"
@@ -3461,9 +3458,6 @@ var AggregationTests = []testdata.AggregationTestCase{
34613458
TestName: "IP range, with ranges as CIDR masks, keyed=true. In Kibana: Add panel > Aggregation Based > Area. Buckets: X-asis: IP Range",
34623459
QueryRequestJson: `
34633460
{
3464-
"_source": {
3465-
"excludes": []
3466-
},
34673461
"aggs": {
34683462
"2": {
34693463
"ip_range": {
@@ -3535,10 +3529,127 @@ var AggregationTests = []testdata.AggregationTestCase{
35353529
}},
35363530
},
35373531
ExpectedPancakeSQL: `
3538-
SELECT countIf("clientip">='255.255.255.254') AS "range_0__aggr__2__count",
3532+
SELECT countIf(("clientip">='255.255.255.254' AND "clientip"<'::1:0:0:0')) AS "range_0__aggr__2__count",
35393533
countIf("clientip">='128.129.130.131') AS "range_1__aggr__2__count",
35403534
countIf(("clientip">='10.0.7.96' AND "clientip"<'10.0.7.128')) AS
35413535
"range_2__aggr__2__count"
35423536
FROM __quesma_table_name`,
35433537
},
3538+
{ // [27]
3539+
TestName: "IP range ipv6",
3540+
QueryRequestJson: `
3541+
{
3542+
"aggs": {
3543+
"2": {
3544+
"ip_range": {
3545+
"field": "clientip",
3546+
"ranges": [
3547+
{
3548+
"from": "1::132:13:21:23:122:22"
3549+
},
3550+
{
3551+
"to": "1::132:13:21:23:122:22"
3552+
},
3553+
{
3554+
"to": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"
3555+
}
3556+
]
3557+
}
3558+
}
3559+
},
3560+
"size": 0,
3561+
"track_total_hits": true
3562+
}`,
3563+
ExpectedResponse: `
3564+
{
3565+
"aggregations": {
3566+
"2": {
3567+
"buckets": [
3568+
{
3569+
"key": "1::132:13:21:23:122:22-*",
3570+
"from": "1::132:13:21:23:122:22",
3571+
"doc_count": 7290
3572+
},
3573+
{
3574+
"key": "*-1::132:13:21:23:122:22",
3575+
"to": "1::132:13:21:23:122:22",
3576+
"doc_count": 6784
3577+
},
3578+
{
3579+
"key": "*-ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
3580+
"to": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff",
3581+
"doc_count": 999999
3582+
}
3583+
]
3584+
}
3585+
}
3586+
}`,
3587+
ExpectedPancakeResults: []model.QueryResultRow{
3588+
{Cols: []model.QueryResultCol{
3589+
model.NewQueryResultCol("range_0__aggr__2__count", int64(7290)),
3590+
model.NewQueryResultCol("range_1__aggr__2__count", int64(6784)),
3591+
model.NewQueryResultCol("range_2__aggr__2__count", int64(999999)),
3592+
}},
3593+
},
3594+
ExpectedPancakeSQL: `
3595+
SELECT countIf("clientip">='1::132:13:21:23:122:22') AS
3596+
"range_0__aggr__2__count",
3597+
countIf("clientip"<'1::132:13:21:23:122:22') AS "range_1__aggr__2__count",
3598+
countIf("clientip"<'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') AS
3599+
"range_2__aggr__2__count"
3600+
FROM __quesma_table_name`,
3601+
},
3602+
{ // [28]
3603+
TestName: "IP range ipv6 with mask",
3604+
QueryRequestJson: `
3605+
{
3606+
"aggs": {
3607+
"2": {
3608+
"ip_range": {
3609+
"field": "clientip",
3610+
"ranges": [
3611+
{
3612+
"mask": "::/2"
3613+
},
3614+
{
3615+
"mask": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/127"
3616+
}
3617+
]
3618+
}
3619+
}
3620+
},
3621+
"size": 0,
3622+
"track_total_hits": true
3623+
}`,
3624+
ExpectedResponse: `
3625+
{
3626+
"aggregations": {
3627+
"2": {
3628+
"buckets": [
3629+
{
3630+
"key": "::/2",
3631+
"to": "4000::",
3632+
"doc_count": 1
3633+
},
3634+
{
3635+
"key": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/127",
3636+
"from": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe",
3637+
"doc_count": 0
3638+
}
3639+
]
3640+
}
3641+
}
3642+
}`,
3643+
ExpectedPancakeResults: []model.QueryResultRow{
3644+
{Cols: []model.QueryResultCol{
3645+
model.NewQueryResultCol("range_0__aggr__2__count", int64(1)),
3646+
model.NewQueryResultCol("range_1__aggr__2__count", int64(0)),
3647+
}},
3648+
},
3649+
ExpectedPancakeSQL: `
3650+
SELECT countIf("clientip"<'4000::') AS "range_0__aggr__2__count",
3651+
countIf("clientip">='ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe') AS
3652+
"range_1__aggr__2__count"
3653+
FROM __quesma_table_name`,
3654+
},
35443655
}

0 commit comments

Comments
 (0)