NormalizeIP for traceroute parser (#1006)

* Rename FixIPv6 for web100 ::: bug
* Move NormalizeIP to parser package
* Use NormalizeIP for sidestream parser
* NormalizeIP for traceroute source and destination

Author: stephen-soltesz

parser/parser.go

@@ -7,6 +7,7 @@ import (
 	"encoding/base64"
 	"fmt"
 	"log"
+	"net"
 	"reflect"
 	"time"
 
@@ -17,6 +18,7 @@ import (
 	"github.com/m-lab/etl/etl"
 	"github.com/m-lab/etl/metrics"
 	"github.com/m-lab/etl/row"
+	"github.com/m-lab/etl/web100"
 )
 
 func init() {
@@ -61,6 +63,21 @@ func GitCommit() string {
 	return gParserGitCommit
 }
 
+// NormalizeIP accepts an IPv4 or IPv6 address and returns a normalized version
+// of that string. This should be used to fix malformed IPv6 addresses in web100
+// datasets (e.g. 2001:::abcd:2) as well as IPv4-mapped IPv6 addresses (e.g. ::ffff:1.2.3.4).
+func NormalizeIP(ip string) string {
+	r, err := web100.FixIPv6(ip)
+	if err != nil {
+		return ip
+	}
+	n := net.ParseIP(r)
+	if n == nil {
+		return r
+	}
+	return n.String()
+}
+
 // NewSinkParser creates an appropriate parser for a given data type.
 // Eventually all datatypes will use this instead of NewParser.
 func NewSinkParser(dt etl.DataType, sink row.Sink, table string, ann api.Annotator) etl.Parser {

parser/parser_test.go

@@ -48,6 +48,52 @@ func (ti *countingInserter) Flush() error {
 	return nil
 }
 
+func TestNormalizeIP(t *testing.T) {
+	tests := []struct {
+		name string
+		ip   string
+		want string
+	}{
+		{
+			name: "success-noop-ipv4",
+			ip:   "1.2.3.4",
+			want: "1.2.3.4",
+		},
+		{
+			name: "success-noop-ipv6",
+			ip:   "1:2:3::4",
+			want: "1:2:3::4",
+		},
+		{
+			name: "success-:::-ipv6",
+			ip:   "1:2:3:::4", // triple-colon format from web100.
+			want: "1:2:3::4",
+		},
+		{
+			name: "badformat-preserved-::::-ipv6",
+			ip:   "1:2:3::::4", // quad-colon format error, not normalized.
+			want: "1:2:3::::4",
+		},
+		{
+			name: "badformat-preserved-corrupt",
+			ip:   "1-2-3-4", // this is not an IP, but b/c it can't be fixed, it's preserved.
+			want: "1-2-3-4",
+		},
+		{
+			name: "success-ipv6-mapped-ipv4",
+			ip:   "::ffff:1.2.3.4", // quad-colon format error, not normalized.
+			want: "1.2.3.4",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := parser.NormalizeIP(tt.ip); got != tt.want {
+				t.Errorf("NormalizeIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 //------------------------------------------------------------------------------------
 // TestParser ignores the content, returns a MapSaver containing meta data and
 // "testname":"..."

parser/pt.go

@@ -304,8 +304,8 @@ func ParseJSONL(testName string, rawContent []byte, tableName string, taskFilena
 		StartTime:      int64(cycleStart.Start_time),
 		StopTime:       int64(cycleStop.Stop_time),
 		ScamperVersion: tracelb.Version,
-		Source:         schema.ServerInfo{IP: tracelb.Src},
-		Destination:    schema.ClientInfo{IP: tracelb.Dst},
+		Source:         schema.ServerInfo{IP: NormalizeIP(tracelb.Src)},
+		Destination:    schema.ClientInfo{IP: NormalizeIP(tracelb.Dst)},
 		ProbeSize:      int64(tracelb.Probe_size),
 		ProbeC:         int64(tracelb.Probec),
 		Hop:            hops,

parser/pt_test.go

@@ -100,11 +100,11 @@ func TestParseJSONLSimple(t *testing.T) {
 	if got.UUID != wantUUID {
 		t.Fatalf("failed to parse UUID, wanted %q, got %q", wantUUID, got.UUID)
 	}
-	wantSourceIP := "::ffff:180.87.97.101"
+	wantSourceIP := "180.87.97.101"
 	if got.Source.IP != wantSourceIP {
 		t.Fatalf("failed to parse source IP, wanted %q, got %q", wantSourceIP, got.Source.IP)
 	}
-	wantDestinationIP := "::ffff:1.47.236.62"
+	wantDestinationIP := "1.47.236.62"
 	if got.Destination.IP != wantDestinationIP {
 		t.Fatalf("failed to parse destination IP, wanted %q, got %q", wantDestinationIP, got.Destination.IP)
 	}

parser/ss.go

@@ -109,8 +109,8 @@ func PackDataIntoSchema(ssValue map[string]string, logTime time.Time, testName s
 		return schema.SS{}, err
 	}
 
-	ssValue["LocalAddress"] = web100.NormalizeIP(ssValue["LocalAddress"])
-	ssValue["RemAddress"] = web100.NormalizeIP(ssValue["RemAddress"])
+	ssValue["LocalAddress"] = NormalizeIP(ssValue["LocalAddress"])
+	ssValue["RemAddress"] = NormalizeIP(ssValue["RemAddress"])
 	connSpec := &schema.Web100ConnectionSpecification{
 		Local_ip:    ssValue["LocalAddress"],
 		Local_af:    web100.ParseIPFamily(ssValue["LocalAddress"]),

web100/parse.go

@@ -69,10 +69,10 @@ var (
 	ErrIPv6QuadColon           = errors.New("IP address contains :::: ")
 )
 
-// NormalizeIPv6 fixes triple colon ::: which is produced by sidestream.
+// FixIPv6 fixes triple colon ::: which is produced by sidestream.
 // This error is produced by older versions of the c-web100 library, which is still
 // used by sidestream.
-func NormalizeIPv6(ipStr string) (string, error) {
+func FixIPv6(ipStr string) (string, error) {
 	split := strings.Split(ipStr, ":::")
 	switch len(split) {
 	case 1:
@@ -87,21 +87,10 @@ func NormalizeIPv6(ipStr string) (string, error) {
 	}
 }
 
-// NormalizeIP accepts an IPv4 or IPv6 address and returns a normalized version
-// of that string. This should be used to fix malformed IPv6 addresses in
-// web100 datasets.
-func NormalizeIP(ip string) string {
-	r, err := NormalizeIPv6(ip)
-	if err != nil {
-		return ip
-	}
-	return r
-}
-
 // ValidateIP validates (and possibly repairs) IP addresses.
 // Return nil if it is a valid IPv4 or IPv6 address (or can be repaired), non-nil otherwise.
 func ValidateIP(ipStr string) error {
-	ipStr, err := NormalizeIPv6(ipStr)
+	ipStr, err := FixIPv6(ipStr)
 	if err != nil {
 		return err
 	}

web100/parse_test.go

@@ -118,39 +118,3 @@ func BenchmarkValidateIPv4(b *testing.B) {
 		_ = web100.ValidateIP("1.2.3.4")
 	}
 }
-
-func TestNormalizeIP(t *testing.T) {
-	tests := []struct {
-		name string
-		ip   string
-		want string
-	}{
-		{
-			name: "success-noop-ipv4",
-			ip:   "1.2.3.4",
-			want: "1.2.3.4",
-		},
-		{
-			name: "success-noop-ipv6",
-			ip:   "1:2:3::4",
-			want: "1:2:3::4",
-		},
-		{
-			name: "success-:::-ipv6",
-			ip:   "1:2:3:::4", // triple-colon format from web100.
-			want: "1:2:3::4",
-		},
-		{
-			name: "badformat-preserved-::::-ipv6",
-			ip:   "1:2:3::::4", // quad-colon format error, not normalized.
-			want: "1:2:3::::4",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if got := web100.NormalizeIP(tt.ip); got != tt.want {
-				t.Errorf("NormalizeIP() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}