Merge pull request #173 from m-lab/parser-fix

Add unit test and fix bugs

Author: gfr10598

parser/ndt_meta.go

@@ -48,7 +48,9 @@ func handleIP(connSpec schema.Web100ValueMap, prefix string, ipString string) {
 	connSpec.SetString(prefix+"_ip", ipString)
 	ip := net.ParseIP(ipString)
 	if ip == nil {
-		// TODO - log/metric
+		log.Printf("Failed parsing connSpec IP: %s\n", ipString)
+		metrics.WarningCount.WithLabelValues(
+			"ndt", "unknown", "failed parsing connSpec IP").Inc()
 	} else {
 		connSpec.SetString(prefix+"_ip", ip.String())
 		if ip.To4() != nil {
@@ -62,18 +64,33 @@ func handleIP(connSpec schema.Web100ValueMap, prefix string, ipString string) {
 func (mfd *MetaFileData) PopulateConnSpec(connSpec schema.Web100ValueMap) {
 	for k, v := range fieldPairs {
 		s, ok := mfd.Fields[k]
-		if ok && s != "" {
-			connSpec.SetString(v, s)
+		if ok {
+			if s != "" {
+				connSpec.SetString(v, s)
+			}
+		} else {
+			log.Printf("Missing field: %s %v\n", k, v)
 		}
 	}
-	s, ok := mfd.Fields["server_ip"]
+	s, ok := connSpec["server_ip"]
 	// TODO - extract function for this stanza
-	if ok && s != "" {
-		handleIP(connSpec, "server", s)
+	if ok {
+		if s != "" {
+			handleIP(connSpec, "server", s.(string))
+		}
+	} else {
+		metrics.WarningCount.WithLabelValues(
+			"table", "unknown", "missing server_ip").Inc()
 	}
-	s, ok = mfd.Fields["client_ip"]
-	if ok && s != "" {
-		handleIP(connSpec, "client", s)
+	s, ok = connSpec["client_ip"]
+	if ok {
+		if s != "" {
+			handleIP(connSpec, "client", s.(string))
+		}
+	} else {
+		log.Println("client_ip missing from .meta")
+		metrics.WarningCount.WithLabelValues(
+			"table", "unknown", "missing client_ip").Inc()
 	}
 }
 

parser/ndt_meta_test.go

@@ -2,10 +2,12 @@ package parser_test
 
 import (
 	"io/ioutil"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/m-lab/etl/parser"
+	"github.com/m-lab/etl/schema"
 )
 
 // Not complete, but verifies basic functionality.
@@ -34,4 +36,32 @@ func TestMetaParser(t *testing.T) {
 	if meta.Fields["server hostname"] != "mlab3.vie01.measurement-lab.org" {
 		t.Error("Incorrect hostname: ", meta.Fields["hostname"])
 	}
+
+	connSpec := schema.EmptyConnectionSpec()
+	meta.PopulateConnSpec(connSpec)
+
+	// This particular file is missing the server_ip address...
+	if _, ok := connSpec["server_ip"]; ok {
+		t.Error("expected server_ip to be empty")
+	}
+
+	// But the client_ip address (and client_af) should be fine.
+	if v, ok := connSpec["client_ip"]; !ok {
+		t.Logf("missing client ip address")
+		for k, v := range meta.Fields {
+			t.Logf("%s : %s\n", k, v)
+		}
+		t.Error("missing client ip address")
+	} else {
+		t.Logf("found client ip: %v\n", v)
+	}
+
+	if v, ok := connSpec["client_af"]; !ok {
+		t.Logf("missing client_af annotation")
+		t.Error("missing client_af")
+	} else {
+		if v.(int64) != syscall.AF_INET {
+			t.Logf("Wrong client_af value: ", v.(int64))
+		}
+	}
 }