Skip to content

Commit 6b71af1

Browse files
cristinaleonrcristinaleonr
authored
Add hopannotation1 schema to legacy ndt/traceroute table schema (#1030)
* Add hopannotation1 schema to legacy ndt/traceroute table schema * build hack * Recreate PCAP table * Add bigquery tag * Change field name to Hostname * Change field to camel case
1 parent 1a3523d commit 6b71af1

File tree

8 files changed

+194
-31
lines changed

8 files changed

+194
-31
lines changed

‎parser/parser.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,18 @@ func NormalizeIP(ip string) string {
7878
return n.String()
7979
}
8080

81+
// GetHopID creates a unique identifier to join Hop Annotations
82+
// with traceroute datasets.
83+
// The same logic exists in traceroute-caller.
84+
// https://github.com/m-lab/traceroute-caller/blob/773bb092b18589d2fee20418ed1fa9aa6c5850cc/triggertrace/triggertrace.go#L147
85+
// https://github.com/m-lab/traceroute-caller/blob/773bb092b18589d2fee20418ed1fa9aa6c5850cc/hopannotation/hopannotation.go#L235
86+
// https://github.com/m-lab/traceroute-caller/blob/773bb092b18589d2fee20418ed1fa9aa6c5850cc/hopannotation/hopannotation.go#L237
87+
func GetHopID(cycleStartTime float64, hostname string, address string) string {
88+
traceStartTime := time.Unix(int64(cycleStartTime), 0).UTC()
89+
date := traceStartTime.Format("20060102")
90+
return fmt.Sprintf("%s_%s_%s", date, hostname, address)
91+
}
92+
8193
// NewSinkParser creates an appropriate parser for a given data type.
8294
// Eventually all datatypes will use this instead of NewParser.
8395
func NewSinkParser(dt etl.DataType, sink row.Sink, table string, ann api.Annotator) etl.Parser {

‎parser/parser_test.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,38 @@ func TestNormalizeIP(t *testing.T) {
112112
}
113113
}
114114

115+
func TestGetHopID(t *testing.T) {
116+
tests := []struct {
117+
name string
118+
cycleStartTime float64
119+
hostname string
120+
address string
121+
want string
122+
}{
123+
{
124+
name: "success",
125+
cycleStartTime: float64(1566691268),
126+
hostname: "ndt-plh7v",
127+
address: "2001:550:3::1ca",
128+
want: "20190825_ndt-plh7v_2001:550:3::1ca",
129+
},
130+
{
131+
name: "unix-start",
132+
cycleStartTime: float64(0),
133+
hostname: "ndt-plh7v",
134+
address: "2001:550:3::1ca",
135+
want: "19700101_ndt-plh7v_2001:550:3::1ca",
136+
},
137+
}
138+
for _, tt := range tests {
139+
t.Run(tt.name, func(t *testing.T) {
140+
if got := parser.GetHopID(tt.cycleStartTime, tt.hostname, tt.address); got != tt.want {
141+
t.Errorf("GetHopID() = %v, want %v", got, tt.want)
142+
}
143+
})
144+
}
145+
}
146+
115147
//------------------------------------------------------------------------------------
116148
// TestParser ignores the content, returns a MapSaver containing meta data and
117149
// "testname":"..."

‎parser/pt.go‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"github.com/m-lab/etl/etl"
2222
"github.com/m-lab/etl/metrics"
2323
"github.com/m-lab/etl/schema"
24+
"github.com/m-lab/traceroute-caller/hopannotation"
2425
)
2526

2627
// -------------------------------------------------
@@ -83,11 +84,11 @@ type ScamperLink struct {
8384
}
8485

8586
type ScamperNode struct {
86-
Addr string `json:"addr"`
87-
Name string `json:"name"`
88-
Q_ttl int `json:"q_ttl"`
89-
Linkc int64 `json:"linkc"`
90-
Links [][]ScamperLink `json:"links"`
87+
Addr string `json:"addr"`
88+
Hostname string `json:"name"`
89+
Q_ttl int `json:"q_ttl"`
90+
Linkc int64 `json:"linkc"`
91+
Links [][]ScamperLink `json:"links"`
9192
}
9293

9394
// There are 4 lines in the traceroute test .jsonl file.
@@ -251,7 +252,7 @@ func ParseJSONL(testName string, rawContent []byte, tableName string, taskFilena
251252
var links []schema.HopLink
252253
if len(oneNode.Links) == 0 {
253254
hops = append(hops, schema.ScamperHop{
254-
Source: schema.HopIP{IP: oneNode.Addr, Hostname: oneNode.Name},
255+
Source: schema.HopIP{IP: oneNode.Addr, Hostname: oneNode.Hostname},
255256
Linkc: oneNode.Linkc,
256257
})
257258
continue
@@ -273,12 +274,15 @@ func ParseJSONL(testName string, rawContent []byte, tableName string, taskFilena
273274
}
274275
links = append(links, schema.HopLink{HopDstIP: oneLink.Addr, TTL: ttl, Probes: probes})
275276
}
277+
278+
hopID := GetHopID(cycleStart.Start_time, cycleStart.Hostname, oneNode.Addr)
279+
hopAnn := &hopannotation.HopAnnotation1{ID: hopID, Timestamp: time.Unix(int64(cycleStart.Start_time), 0).UTC()}
276280
hops = append(hops, schema.ScamperHop{
277-
Source: schema.HopIP{IP: oneNode.Addr, Hostname: oneNode.Name},
278-
Linkc: oneNode.Linkc,
279-
Links: links,
281+
Source: schema.HopIP{IP: oneNode.Addr, Hostname: oneNode.Hostname,
282+
HopAnnotation1: hopAnn},
283+
Linkc: oneNode.Linkc,
284+
Links: links,
280285
})
281-
282286
}
283287

284288
err = json.Unmarshal([]byte(jsonStrings[3]), &cycleStop)

‎parser/pt_test.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
v2 "github.com/m-lab/annotation-service/api/v2"
1515
"github.com/m-lab/etl/parser"
1616
"github.com/m-lab/etl/schema"
17+
"github.com/m-lab/traceroute-caller/hopannotation"
1718
)
1819

1920
func TestParsePT(t *testing.T) {
@@ -169,8 +170,10 @@ func TestParseJSONLComplex(t *testing.T) {
169170
}
170171

171172
wantHop := schema.ScamperHop{
172-
Source: schema.HopIP{IP: "2001:550:1b01:1::1", ASN: 0},
173-
Linkc: 1,
173+
Source: schema.HopIP{IP: "2001:550:1b01:1::1", ASN: 0,
174+
HopAnnotation1: &hopannotation.HopAnnotation1{ID: "20190825_ndt-plh7v_2001:550:1b01:1::1",
175+
Timestamp: time.Date(2019, time.August, 25, 00, 01, 8, 0, time.UTC)}},
176+
Linkc: 1,
174177
Links: []schema.HopLink{
175178
schema.HopLink{
176179
HopDstIP: "2001:550:3::1ca",

‎parser/scamper1.go‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,6 @@ func NewScamper1Parser(sink row.Sink, table, suffix string, ann v2as.Annotator)
4444
}
4545
}
4646

47-
// GetTraceStartDate extracts the date portion of the cycle-start timestamp in YYYYMMDD format.
48-
func GetTraceStartDate(cycleStartTime float64) string {
49-
traceStartTime := time.Unix(int64(cycleStartTime), 0).UTC()
50-
date := traceStartTime.Format("20060102")
51-
return date
52-
}
53-
5447
// parseTracelb parses the TracelbLine struct defined in traceroute-caller and populates the BQTracelbLine.
5548
func parseTracelb(bqScamperOutput *schema.BQScamperOutput, tracelb parser.TracelbLine) {
5649
bqScamperOutput.Tracelb = schema.BQTracelbLine{
@@ -77,8 +70,6 @@ func parseTracelb(bqScamperOutput *schema.BQScamperOutput, tracelb parser.Tracel
7770

7871
nodes := tracelb.Nodes
7972
bqScamperOutput.Tracelb.Nodes = make([]schema.BQScamperNode, 0, len(nodes))
80-
date := GetTraceStartDate(bqScamperOutput.CycleStart.StartTime)
81-
hostname := bqScamperOutput.CycleStart.Hostname
8273

8374
for _, node := range nodes {
8475
bqLinkArray := make([]schema.BQScamperLinkArray, 0, len(node.Links))
@@ -90,7 +81,8 @@ func parseTracelb(bqScamperOutput *schema.BQScamperOutput, tracelb parser.Tracel
9081
}
9182

9283
bqScamperNode := schema.BQScamperNode{
93-
HopID: fmt.Sprintf("%s_%s_%s", date, hostname, node.Addr),
84+
HopID: GetHopID(bqScamperOutput.CycleStart.StartTime, bqScamperOutput.CycleStart.Hostname,
85+
node.Addr),
9486
Addr: node.Addr,
9587
Name: node.Name,
9688
QTTL: node.QTTL,

‎parser/scamper1_test.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package parser_test
22

33
import (
4-
"fmt"
54
"io/ioutil"
65
"path"
76
"strings"
@@ -217,17 +216,16 @@ func expectedScamper1Row() schema.Scamper1Row {
217216
}
218217

219218
cycleStartTime := float64(1566691268)
220-
cycleStartDate := parser.GetTraceStartDate(float64(cycleStartTime))
221219
bqScamperNode1 := schema.BQScamperNode{
222-
HopID: fmt.Sprintf("%s_ndt-plh7v_2001:550:1b01:1::1", cycleStartDate),
220+
HopID: parser.GetHopID(cycleStartTime, "ndt-plh7v", "2001:550:1b01:1::1"),
223221
Addr: "2001:550:1b01:1::1",
224222
Name: "",
225223
QTTL: 1,
226224
Linkc: 1,
227225
Links: bqScamperLinkArray1,
228226
}
229227
bqScamperNode2 := schema.BQScamperNode{
230-
HopID: fmt.Sprintf("%s_ndt-plh7v_2001:4888:3f:6092:3a2:26:0:1", cycleStartDate),
228+
HopID: parser.GetHopID(cycleStartTime, "ndt-plh7v", "2001:4888:3f:6092:3a2:26:0:1"),
231229
Addr: "2001:4888:3f:6092:3a2:26:0:1",
232230
Name: "",
233231
QTTL: 1,

‎schema/pt_schema.go‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,19 @@ import (
99
"github.com/m-lab/annotation-service/api"
1010
v2as "github.com/m-lab/annotation-service/api/v2"
1111
"github.com/m-lab/go/cloud/bqx"
12+
"github.com/m-lab/traceroute-caller/hopannotation"
1213
"github.com/m-lab/uuid-annotator/annotator"
1314

1415
"github.com/m-lab/etl/metrics"
1516
)
1617

1718
type HopIP struct {
18-
IP string `json:"ip"`
19-
City string `json:"city"`
20-
CountryCode string `json:"country_code"`
21-
Hostname string `json:"hostname"`
22-
ASN uint32 `json:"asn,uint32"`
19+
IP string `json:"ip" bigquery:"IP"`
20+
City string `json:"city" bigquery:"City"`
21+
CountryCode string `json:"country_code" bigquery:"CountryCode"`
22+
Hostname string `json:"hostname" bigquery:"Hostname"`
23+
ASN uint32 `json:"asn,uint32" bigquery:"ASN"`
24+
HopAnnotation1 *hopannotation.HopAnnotation1 `json:"hopannotation1" bigquery:"HopAnnotation1"`
2325
}
2426

2527
type HopProbe struct {
@@ -99,6 +101,7 @@ func (row *PTTest) GetServerIP() string {
99101
return row.Source.IP
100102
}
101103

104+
// AnnotateHops adds geolocation and network information to the hops.
102105
func (row *PTTest) AnnotateHops(annMap map[string]*api.Annotations) error {
103106
for index, _ := range row.Hop {
104107
ann, ok := annMap[row.Hop[index].Source.IP]
@@ -121,6 +124,10 @@ func (row *PTTest) AnnotateHops(annMap map[string]*api.Annotations) error {
121124
}
122125
row.Hop[index].Source.ASN = uint32(asn)
123126
}
127+
128+
if row.Hop[index].Source.HopAnnotation1 != nil {
129+
row.Hop[index].Source.HopAnnotation1.Annotations = v2as.ConvertAnnotationsToClientAnnotations(ann)
130+
}
124131
}
125132
return nil
126133
}

‎schema/pt_schema_test.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
package schema_test
2+
3+
import (
4+
"reflect"
5+
"testing"
6+
7+
"github.com/m-lab/annotation-service/api"
8+
"github.com/m-lab/etl/schema"
9+
"github.com/m-lab/traceroute-caller/hopannotation"
10+
"github.com/m-lab/uuid-annotator/annotator"
11+
)
12+
13+
func TestAnnotateHops(t *testing.T) {
14+
testMap := map[string]*api.Annotations{
15+
"91.213.30.229": {
16+
Geo: &api.GeolocationIP{
17+
ContinentCode: "NA",
18+
CountryCode: "US",
19+
City: "NYC",
20+
Latitude: 1.0,
21+
Longitude: 2.0,
22+
},
23+
Network: &api.ASData{
24+
ASNumber: 1234,
25+
Systems: []api.System{
26+
{ASNs: []uint32{1234}},
27+
},
28+
},
29+
},
30+
"91.169.126.135": {
31+
Geo: &api.GeolocationIP{
32+
ContinentCode: "EU",
33+
CountryCode: "DE",
34+
Latitude: 3.0,
35+
Longitude: 4.0,
36+
},
37+
Network: &api.ASData{
38+
ASNumber: 4321,
39+
Systems: []api.System{
40+
{ASNs: []uint32{4321}},
41+
},
42+
},
43+
},
44+
}
45+
testHops := []schema.ScamperHop{{
46+
Source: schema.HopIP{
47+
IP: "91.213.30.229",
48+
HopAnnotation1: &hopannotation.HopAnnotation1{},
49+
}}, {
50+
Source: schema.HopIP{
51+
IP: "91.169.126.135",
52+
}},
53+
}
54+
tests := []struct {
55+
name string
56+
hops []schema.ScamperHop
57+
annMap map[string]*api.Annotations
58+
wantHops []schema.ScamperHop
59+
}{
60+
{
61+
name: "empty-ann-map",
62+
hops: testHops,
63+
annMap: map[string]*api.Annotations{},
64+
wantHops: testHops,
65+
},
66+
{
67+
name: "valid-input",
68+
hops: testHops,
69+
annMap: testMap,
70+
wantHops: []schema.ScamperHop{{
71+
Source: schema.HopIP{
72+
IP: "91.213.30.229",
73+
City: "NYC",
74+
CountryCode: "US",
75+
ASN: 1234,
76+
HopAnnotation1: &hopannotation.HopAnnotation1{
77+
Annotations: &annotator.ClientAnnotations{
78+
Geo: &annotator.Geolocation{
79+
ContinentCode: "NA",
80+
CountryCode: "US",
81+
City: "NYC",
82+
Latitude: 1,
83+
Longitude: 2,
84+
},
85+
Network: &annotator.Network{
86+
ASNumber: 1234,
87+
Systems: []annotator.System{
88+
{ASNs: []uint32{uint32(1234)}},
89+
},
90+
},
91+
},
92+
},
93+
}}, {
94+
Source: schema.HopIP{
95+
IP: "91.169.126.135",
96+
CountryCode: "DE",
97+
ASN: 4321,
98+
}},
99+
},
100+
},
101+
}
102+
103+
for _, tt := range tests {
104+
t.Run(tt.name, func(t *testing.T) {
105+
row := schema.PTTest{}
106+
row.Hop = tt.hops
107+
annMap := tt.annMap
108+
row.AnnotateHops(annMap)
109+
110+
if !reflect.DeepEqual(row.Hop, tt.wantHops) {
111+
t.Fatalf("failed to annotate hops,\nwanted: %+v\ngot: %+v", tt.wantHops, row.Hop)
112+
}
113+
})
114+
}
115+
}

0 commit comments

Comments
 (0)