Skip to content

Commit 3b3f037

Browse files
authored
Merge pull request #467 from m-lab/ipv6
Add IPv6 triple colon fix
2 parents 7dc12ee + 26ad1cf commit 3b3f037

File tree

6 files changed

+166
-129
lines changed

6 files changed

+166
-129
lines changed

parser/ndt_meta.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313

1414
"github.com/m-lab/etl/metrics"
1515
"github.com/m-lab/etl/schema"
16+
"github.com/m-lab/etl/web100"
1617
)
1718

1819
// MetaFileData is the parsed info from the .meta file.
@@ -55,13 +56,13 @@ var fieldPairs = map[string]string{
5556

5657
func handleIP(connSpec schema.Web100ValueMap, prefix string, ipString string) {
5758
connSpec.SetString(prefix+"_ip", ipString)
58-
if ValidateIP(ipString) != nil {
59+
if web100.ValidateIP(ipString) != nil {
5960
log.Printf("Failed parsing connSpec IP: %s\n", ipString)
6061
metrics.WarningCount.WithLabelValues(
6162
"ndt", "unknown", "failed parsing connSpec IP").Inc()
6263
} else {
6364
connSpec.SetString(prefix+"_ip", ipString)
64-
family := ParseIPFamily(ipString)
65+
family := web100.ParseIPFamily(ipString)
6566
if family != -1 {
6667
connSpec.SetInt64(prefix+"_af", family)
6768
}

parser/ss.go

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ package parser
44
import (
55
"bytes"
66
"errors"
7-
"fmt"
87
"log"
98
"path/filepath"
109
"reflect"
@@ -106,7 +105,7 @@ func PackDataIntoSchema(ss_value map[string]string, log_time time.Time, testName
106105

107106
conn_spec := &schema.Web100ConnectionSpecification{
108107
Local_ip: ss_value["LocalAddress"],
109-
Local_af: ParseIPFamily(ss_value["LocalAddress"]),
108+
Local_af: web100.ParseIPFamily(ss_value["LocalAddress"]),
110109
Local_port: int64(local_port),
111110
Remote_ip: ss_value["RemAddress"],
112111
Remote_port: int64(remote_port),
@@ -206,16 +205,15 @@ func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName stri
206205
metrics.WorkerState.WithLabelValues(ss.TableName(), "ss").Inc()
207206
defer metrics.WorkerState.WithLabelValues(ss.TableName(), "ss").Dec()
208207

209-
log_time, err := ExtractLogtimeFromFilename(testName)
208+
logTime, err := ExtractLogtimeFromFilename(testName)
210209
if err != nil {
211210
return err
212211
}
213-
var var_names []string
214212
testContent := strings.Split(string(rawContent[:]), "\n")
215213
if len(testContent) < 2 {
216-
return errors.New("empty test file.")
214+
return errors.New("empty test file")
217215
}
218-
var_names, err = ParseKHeader(testContent[0])
216+
varNames, err := ParseKHeader(testContent[0])
219217
if err != nil {
220218
metrics.ErrorCount.WithLabelValues(
221219
ss.TableName(), "ss", "corrupted header").Inc()
@@ -227,34 +225,34 @@ func (ss *SSParser) ParseAndInsert(meta map[string]bigquery.Value, testName stri
227225
if len(oneLine) == 0 {
228226
continue
229227
}
230-
ss_value, err := ParseOneLine(oneLine, var_names)
228+
ssValue, err := ParseOneLine(oneLine, varNames)
231229
if err != nil {
232230
metrics.TestCount.WithLabelValues(
233231
ss.TableName(), "ss", "corrupted content").Inc()
234-
return err
232+
continue
235233
}
236-
err = ValidateIP(ss_value["LocalAddress"])
234+
err = web100.ValidateIP(ssValue["LocalAddress"])
237235
if err != nil {
238236
metrics.TestCount.WithLabelValues(
239237
ss.TableName(), "ss", "Invalid server IP").Inc()
240-
return fmt.Errorf("Invalid server IP address: %s with error: %s", ss_value["LocalAddress"], err)
238+
log.Printf("Invalid server IP address: %s with error: %s\n", ssValue["LocalAddress"], err)
239+
continue
241240
}
242-
err = ValidateIP(ss_value["RemAddress"])
241+
err = web100.ValidateIP(ssValue["RemAddress"])
243242
if err != nil {
244243
metrics.TestCount.WithLabelValues(
245244
ss.TableName(), "ss", "Invalid client IP").Inc()
246-
return fmt.Errorf("Invalid client IP address: %s with error: %s", ss_value["RemAddress"], err)
245+
log.Printf("Invalid client IP address: %s with error: %s", ssValue["RemAddress"], err)
246+
continue
247247
}
248-
ss_test, err := PackDataIntoSchema(ss_value, log_time, testName)
248+
ssTest, err := PackDataIntoSchema(ssValue, logTime, testName)
249249
if err != nil {
250-
metrics.ErrorCount.WithLabelValues(
251-
ss.TableName(), "ss", "corrupted data").Inc()
252250
metrics.TestCount.WithLabelValues(
253251
ss.TableName(), "ss", "corrupted data").Inc()
254252
log.Printf("cannot pack data into sidestream schema: %v\n", err)
255-
return err
253+
continue
256254
}
257-
err = ss.inserter.InsertRow(ss_test)
255+
err = ss.inserter.InsertRow(ssTest)
258256
if err != nil {
259257
metrics.ErrorCount.WithLabelValues(
260258
ss.TableName(), "ss", "insert-err").Inc()

parser/util.go

Lines changed: 0 additions & 64 deletions
This file was deleted.

parser/util_test.go

Lines changed: 0 additions & 46 deletions
This file was deleted.

web100/parse.go

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package web100
22

33
import (
4+
"errors"
45
"io"
56
"io/ioutil"
7+
"net"
68
"strings"
9+
"syscall"
710
)
811

912
// ParseWeb100Definitions reads all web100 variable definitions from tcpKis and
@@ -38,3 +41,102 @@ func ParseWeb100Definitions(tcpKis io.Reader) (map[string]string, error) {
3841
}
3942
return legacyNamesToNewNames, nil
4043
}
44+
45+
// ParseIPFamily determines whether an IP string is v4 or v6
46+
func ParseIPFamily(ipStr string) int64 {
47+
ip := net.ParseIP(ipStr)
48+
if ip.To4() != nil {
49+
return syscall.AF_INET
50+
} else if ip.To16() != nil {
51+
return syscall.AF_INET6
52+
}
53+
return -1
54+
}
55+
56+
// IP validation errors.
57+
var (
58+
ErrIPIsUnparseable = errors.New("IP not parsable")
59+
ErrIPIsUnconvertible = errors.New("IP not convertible to ipv4 or ipv6")
60+
ErrIPIsZero = errors.New("IP is zero/unspecified")
61+
ErrIPIsUnroutable = errors.New("IP is nonroutable")
62+
63+
ErrIPv4IsPrivate = errors.New("private IPv4")
64+
ErrIPv6IsPrivate = errors.New("private IPv6")
65+
ErrIPv4IsUnroutable = errors.New("unroutable IPv4")
66+
ErrIPv6IsUnroutable = errors.New("unroutable IPv6")
67+
68+
ErrIPv6MultipleTripleColon = errors.New("more than one ::: in an ip address")
69+
ErrIPv6QuadColon = errors.New("IP address contains :::: ")
70+
)
71+
72+
// NormalizeIPv6 fixes triple colon ::: which is produced by sidestream.
73+
// This error is produced by older versions of the c-web100 library, which is still
74+
// used by sidestream.
75+
func NormalizeIPv6(ipStr string) (string, error) {
76+
split := strings.Split(ipStr, ":::")
77+
switch len(split) {
78+
case 1:
79+
return ipStr, nil
80+
case 2:
81+
if split[1][0] == ':' {
82+
return "", ErrIPv6QuadColon
83+
}
84+
return split[0] + "::" + split[1], nil
85+
default:
86+
return ipStr, ErrIPv6MultipleTripleColon
87+
}
88+
}
89+
90+
// ValidateIP validates (and possibly repairs) IP addresses.
91+
// Return nil if it is a valid IPv4 or IPv6 address (or can be repaired), non-nil otherwise.
92+
func ValidateIP(ipStr string) error {
93+
ipStr, err := NormalizeIPv6(ipStr)
94+
if err != nil {
95+
return err
96+
}
97+
ip := net.ParseIP(ipStr)
98+
if ip == nil || (ip.To4() == nil && ip.To16() == nil) {
99+
return ErrIPIsUnparseable
100+
}
101+
if ip.To4() == nil && ip.To16() == nil {
102+
return ErrIPIsUnconvertible
103+
}
104+
if ip.IsUnspecified() {
105+
return ErrIPIsZero
106+
}
107+
108+
if ip.IsLoopback() || ip.IsMulticast() || ip.IsLinkLocalUnicast() {
109+
return ErrIPIsUnroutable
110+
}
111+
112+
if ip.To4() != nil {
113+
// Check whether it is a private IP.
114+
_, private24BitBlock, _ := net.ParseCIDR("10.0.0.0/8")
115+
_, private20BitBlock, _ := net.ParseCIDR("172.16.0.0/12")
116+
_, private16BitBlock, _ := net.ParseCIDR("192.168.0.0/16")
117+
_, private22BitBlock, _ := net.ParseCIDR("100.64.0.0/10")
118+
if private24BitBlock.Contains(ip) || private20BitBlock.Contains(ip) ||
119+
private16BitBlock.Contains(ip) || private22BitBlock.Contains(ip) {
120+
return ErrIPv4IsPrivate
121+
}
122+
123+
// check whether it is nonroutable IP
124+
_, nonroutable1, _ := net.ParseCIDR("0.0.0.0/8")
125+
_, nonroutable2, _ := net.ParseCIDR("192.0.2.0/24")
126+
if nonroutable1.Contains(ip) || nonroutable2.Contains(ip) {
127+
return ErrIPv4IsUnroutable
128+
}
129+
} else if ip.To16() != nil {
130+
_, private7BitBlock, _ := net.ParseCIDR("FC00::/7")
131+
if private7BitBlock.Contains(ip) {
132+
return ErrIPv6IsPrivate
133+
}
134+
135+
_, nonroutable1, _ := net.ParseCIDR("2001:db8::/32")
136+
_, nonroutable2, _ := net.ParseCIDR("fec0::/10")
137+
if nonroutable1.Contains(ip) || nonroutable2.Contains(ip) {
138+
return ErrIPv6IsUnroutable
139+
}
140+
}
141+
return nil
142+
}

web100/parse_test.go

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package web100_test
22

33
import (
44
"bytes"
5+
"fmt"
6+
"syscall"
57
"testing"
68

79
"github.com/m-lab/etl/web100"
@@ -72,3 +74,47 @@ func TestParseWeb100Definitions(t *testing.T) {
7274
}
7375
}
7476
}
77+
78+
func TestParseIPFamily(t *testing.T) {
79+
if web100.ParseIPFamily("1.2.3.4") != syscall.AF_INET {
80+
t.Fatalf("IPv4 address not parsed correctly.")
81+
}
82+
if web100.ParseIPFamily("2001:db8:0:1:1:1:1:1") != syscall.AF_INET6 {
83+
t.Fatalf("IPv6 address not parsed correctly.")
84+
}
85+
}
86+
87+
func TestValidateIP(t *testing.T) {
88+
if web100.ValidateIP("1.2.3.4") != nil {
89+
fmt.Println(web100.ValidateIP("1.2.3.4"))
90+
t.Fatalf("Valid IPv4 was identified as invalid.")
91+
}
92+
if web100.ValidateIP("2620:0:1000:2304:8053:fe91:6e2e:b4f1") != nil {
93+
t.Fatalf("Valid IPv6 was identified as invalid.")
94+
}
95+
if web100.ValidateIP("::") == nil || web100.ValidateIP("0.0.0.0") == nil ||
96+
web100.ValidateIP("abc.0.0.0") == nil || web100.ValidateIP("1.0.0.256") == nil {
97+
t.Fatalf("Invalid IP was identified as valid.")
98+
}
99+
if web100.ValidateIP("172.16.0.1") == nil {
100+
t.Fatalf("Private IP was not identified as invalid IP.")
101+
}
102+
if web100.ValidateIP("127.0.0.1") == nil || web100.ValidateIP("::ffff:127.0.0.1") == nil {
103+
t.Fatalf("Nonroutable IP was not identified as invalid IP.")
104+
}
105+
106+
if web100.ValidateIP("2001:668:1f:22:::81") != nil {
107+
t.Fatalf("IPv6 with triple colon was not repaired.")
108+
}
109+
if web100.ValidateIP("2001:668:1f::::81") == nil {
110+
t.Fatalf("IPv6 with quad colon was allowed.")
111+
}
112+
}
113+
114+
// To run benchmark...
115+
// go test -bench=. ./parser/...
116+
func BenchmarkValidateIPv4(b *testing.B) {
117+
for i := 0; i < b.N; i++ {
118+
_ = web100.ValidateIP("1.2.3.4")
119+
}
120+
}

0 commit comments

Comments
 (0)