Reset Region field for client and server Geo annotations (#1073)

* Add client and server Region reset
* Add annotation with empty geo fields for unit tests.
* Assert that Client.* and Server.Geo.Region are cleared in test

Author: stephen-soltesz

parser/annotation.go

@@ -100,6 +100,17 @@ func (ap *AnnotationParser) ParseAndInsert(meta map[string]bigquery.Value, testN
 	row.UUID = raw.UUID
 	row.Server = raw.Server
 	row.Client = raw.Client
+
+	// NOTE: Due to https://github.com/m-lab/etl/issues/1069, we mask the Region
+	// field found in synthetic uuid annotations prior to 2020-03-12, and no
+	// longer found in later Geo2 annotations.
+	if row.Server.Geo != nil {
+		row.Server.Geo.Region = ""
+	}
+	if row.Client.Geo != nil {
+		row.Client.Geo.Region = ""
+	}
+
 	// NOTE: annotations are joined with other tables using the UUID, so
 	// finegrain timestamp is not necessary.
 	//

parser/annotation_test.go

@@ -23,6 +23,10 @@ func TestAnnotationParser_ParseAndInsert(t *testing.T) {
 			name: "success",
 			file: "ndt-njp6l_1585004303_00000000000170FA.json",
 		},
+		{
+			name: "success-empty-geo",
+			file: "ndt-empty-geo.json",
+		},
 		{
 			name:    "corrupt-input",
 			file:    "ndt-corrupt.json",
@@ -57,15 +61,22 @@ func TestAnnotationParser_ParseAndInsert(t *testing.T) {
 				expPI := schema.ParseInfo{
 					Version:    "https://github.com/m-lab/etl/tree/foobar",
 					Time:       row.Parser.Time,
-					ArchiveURL: "gs://mlab-test-bucket/ndt/ndt7/2020/03/18/ndt-njp6l_1585004303_00000000000170FA.json",
-					Filename:   "ndt-njp6l_1585004303_00000000000170FA.json",
+					ArchiveURL: "gs://mlab-test-bucket/ndt/ndt7/2020/03/18/" + tt.file,
+					Filename:   tt.file,
 					Priority:   0,
 					GitCommit:  "12345678",
 				}
 
 				if diff := deep.Equal(row.Parser, expPI); diff != nil {
 					t.Errorf("AnnotationParser.ParseAndInsert() different summary: %s", strings.Join(diff, "\n"))
 				}
+
+				if row.Client.Geo != nil && row.Client.Geo.Region != "" {
+					t.Errorf("AnnotationParser.ParseAndInsert() did not clear Client.Geo.Region: %q", row.Client.Geo.Region)
+				}
+				if row.Server.Geo != nil && row.Server.Geo.Region != "" {
+					t.Errorf("AnnotationParser.ParseAndInsert() did not clear Server.Geo.Region: %q", row.Server.Geo.Region)
+				}
 			}
 		})
 	}

parser/testdata/Annotation/ndt-empty-geo.json

@@ -0,0 +1,32 @@
+{
+  "UUID": "ndt-njp6l_1585004303_00000000000170FA",
+  "Timestamp": "2020-04-03T00:00:02.663132563Z",
+  "Server": {
+    "Site": "lga1t",
+    "Machine": "mlab2",
+    "Network": {
+      "ASNumber": 3356,
+      "ASName": "Level 3 Parent, LLC",
+      "Systems": [
+        {
+          "ASNs": [
+            3356
+          ]
+        }
+      ]
+    }
+  },
+  "Client": {
+    "Network": {
+      "CIDR": "35.184.0.0/13",
+      "ASNumber": 15169,
+      "Systems": [
+        {
+          "ASNs": [
+            15169
+          ]
+        }
+      ]
+    }
+  }
+}

parser/testdata/Annotation/ndt-njp6l_1585004303_00000000000170FA.json

@@ -7,6 +7,7 @@
     "Geo": {
       "ContinentCode": "NA",
       "CountryCode": "US",
+      "Region": "should-be-cleared",
       "City": "New York",
       "Latitude": 40.7667,
       "Longitude": -73.8667
@@ -28,6 +29,7 @@
       "ContinentCode": "NA",
       "CountryCode": "US",
       "CountryName": "United States",
+      "Region": "should-be-cleared",
       "Subdivision1ISOCode": "VA",
       "Subdivision1Name": "Virginia",
       "Latitude": 38.6583,
@@ -46,4 +48,4 @@
       ]
     }
   }
-}
+}