Merge pull request #172 from m-lab/soltesz-fix-hostname

Add missing NDT client values & Add tls & websocket columns

Author: stephen-soltesz

parser/ndt_meta.go

@@ -42,6 +42,17 @@ var fieldPairs = map[string]string{
 	"client OS name":          "client_os",
 	"client_browser name":     "client_browser",
 	"client_application name": "client_application",
+
+	// Some client fields are "Additional" meta data optionally provided by the client.
+	// The NDT client names these fields differently than the server.
+	// Other clients may provide different key names.
+	"client.kernel.version": "client_kernel_version",
+	"client.version":        "client_version",
+
+	// NDT SSL added two additional meta fields to signify whether the test was
+	// a websocket and/or tls test.
+	"tls":        "tls",
+	"websockets": "websockets",
 }
 
 func handleIP(connSpec schema.Web100ValueMap, prefix string, ipString string) {
@@ -72,6 +83,17 @@ func (mfd *MetaFileData) PopulateConnSpec(connSpec schema.Web100ValueMap) {
 			log.Printf("Missing field: %s %v\n", k, v)
 		}
 	}
+	// Only set the value for tls & websocket if the field is present.
+	if s, ok := mfd.Fields["tls"]; ok {
+		if s != "" {
+			connSpec.SetBool("tls", mfd.Tls)
+		}
+	}
+	if s, ok := mfd.Fields["websockets"]; ok {
+		if s != "" {
+			connSpec.SetBool("websockets", mfd.Websockets)
+		}
+	}
 	s, ok := connSpec["server_ip"]
 	// TODO - extract function for this stanza
 	if ok {
@@ -80,7 +102,7 @@ func (mfd *MetaFileData) PopulateConnSpec(connSpec schema.Web100ValueMap) {
 		}
 	} else {
 		metrics.WarningCount.WithLabelValues(
-			"table", "unknown", "missing server_ip").Inc()
+			"ndt", "unknown", "missing server_ip").Inc()
 	}
 	s, ok = connSpec["client_ip"]
 	if ok {
@@ -90,7 +112,7 @@ func (mfd *MetaFileData) PopulateConnSpec(connSpec schema.Web100ValueMap) {
 	} else {
 		log.Println("client_ip missing from .meta")
 		metrics.WarningCount.WithLabelValues(
-			"table", "unknown", "missing client_ip").Inc()
+			"ndt", "unknown", "missing client_ip").Inc()
 	}
 }
 
@@ -110,8 +132,10 @@ func createMetaFileData(testName string, fields map[string]string) (*MetaFileDat
 				"20060102T15:04:05.999999999Z", v)
 		case "tls":
 			data.Tls, err = strconv.ParseBool(v)
+			data.Fields[k] = v
 		case "websockets":
 			data.Websockets, err = strconv.ParseBool(v)
+			data.Fields[k] = v
 		case "Summary data":
 			err = json.Unmarshal(
 				[]byte(`{"SummaryData":[`+v+`]}`),

parser/ndt_meta_test.go

@@ -37,6 +37,7 @@ func TestMetaParser(t *testing.T) {
 		t.Error("Incorrect hostname: ", meta.Fields["hostname"])
 	}
 
+	// Check for the presenece of select connSpec fields.
 	connSpec := schema.EmptyConnectionSpec()
 	meta.PopulateConnSpec(connSpec)
 
@@ -63,5 +64,25 @@ func TestMetaParser(t *testing.T) {
 		if v.(int64) != syscall.AF_INET {
 			t.Logf("Wrong client_af value: ", v.(int64))
 		}
+
+	}
+
+	if s, ok := connSpec["tls"]; ok {
+		t.Errorf("Found field tls: got %q; want not found", s)
+	}
+	if connSpec["websockets"] != true {
+		t.Errorf("Incorrect websockets: got %q; want true", connSpec["websockets"])
+	}
+	if connSpec["client_kernel_version"] != "3.14.0" {
+		t.Errorf("Incorrect client_kernel_version: got %s; want 3.14.0",
+			connSpec["client_kernel_version"])
+	}
+	if connSpec["client_version"] != "3.7.0" {
+		t.Errorf("Incorrect client_version: got %s; want 3.7.0",
+			connSpec["client_version"])
+	}
+	if connSpec["client_os"] != "CLIWebsockets" {
+		t.Errorf("Incorrect client_os: got %s; want CLIWebsockets",
+			connSpec["client_os"])
 	}
 }

parser/testdata/20170509T13:45:13.590210000Z_eb.measurementlab.net:53000.meta

@@ -16,5 +16,6 @@ client_application name:
 Summary data: 0,36,1346,14,2983,24,32,2,74,0,1448,16,39,136448,23184,8688,0,10521978,347602,105334,0,4,4,2896,364,136448,100,0,0,0,1,6,3,2,10,2,74,97,7,22,6,0,156,0,-1,-1,0,1,0,-1,1448,8688,7
  * Additional data:
 client.os.name: CLIWebsockets
-tls: false
+client.version: 3.7.0
+client.kernel.version: 3.14.0
 websockets: true

schema/ndt.json

@@ -26,6 +26,8 @@
           { "name": "server_hostname", "type": "STRING"},
           { "name": "server_ip", "type": "STRING"},
           { "name": "server_kernel_version", "type": "STRING"},
+          { "name": "tls", "type": "BOOLEAN"},
+          { "name": "websockets", "type": "BOOLEAN"},
           {
             "fields": [
               { "name": "area_code", "type": "INTEGER"},

schema/web100.go

@@ -85,6 +85,11 @@ func (s Web100ValueMap) SetString(name string, value string) {
 	s[name] = value
 }
 
+// SetBool saves a boolean in a field with the given name.
+func (s Web100ValueMap) SetBool(name string, value bool) {
+	s[name] = value
+}
+
 // if overwrite is false, will only add missing values.
 // if overwrite is true, will overwrite existing values.
 func (r Web100ValueMap) SubstituteString(overwrite bool, target []string, source []string) {
@@ -179,6 +184,8 @@ func FullConnectionSpec() Web100ValueMap {
 		"client_browser":        "",
 		"client_application":    "",
 		"data_direction":        0,
+		"tls":                   false,
+		"websockets":            false,
 		"client_geolocation":    FullGeolocation(),
 		"server_geolocation":    FullGeolocation(),
 	}

web100/web100.go

@@ -81,6 +81,7 @@ import (
 type Saver interface {
 	SetInt64(name string, value int64)
 	SetString(name string, value string)
+	SetBool(name string, value bool)
 }
 
 //=================================================================================