Skip to content

Commit b5ab8e8

Browse files
stephen-solteszstephen-soltesz
authored
Add NDTWeb100 struct to mirror current BQ schema (#988)
* Add ndt_web100 struct to mirror current BQ schema * Add ndt web100 schema support to update-schema
1 parent 7c6ad82 commit b5ab8e8

File tree

2 files changed

+242
-0
lines changed

2 files changed

+242
-0
lines changed

‎cmd/update-schema/update.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@ func CreateOrUpdateSS(project string, dataset string, table string) error {
5050

5151
return CreateOrUpdate(schema, project, dataset, table, "")
5252
}
53+
func CreateOrUpdateNDTWeb100(project string, dataset string, table string) error {
54+
row := schema.NDTWeb100{}
55+
schema, err := row.Schema()
56+
rtx.Must(err, "NDTWeb100.Schema")
57+
58+
return CreateOrUpdate(schema, project, dataset, table, "")
59+
}
5360

5461
func CreateOrUpdateNDT5ResultRow(project string, dataset string, table string) error {
5562
row := schema.NDT5ResultRow{}
@@ -184,6 +191,12 @@ func updateLegacyTables(project string) int {
184191
if err := CreateOrUpdateSS(project, "batch", "sidestream"); err != nil {
185192
errCount++
186193
}
194+
if err := CreateOrUpdateNDTWeb100(project, "base_tables", "ndt"); err != nil {
195+
errCount++
196+
}
197+
if err := CreateOrUpdateNDTWeb100(project, "batch", "ndt"); err != nil {
198+
errCount++
199+
}
187200
if err := CreateOrUpdateNDT5ResultRow(project, "base_tables", "ndt5"); err != nil {
188201
errCount++
189202
}
@@ -253,6 +266,13 @@ func main() {
253266
if err := CreateOrUpdateSS(*project, "batch", "sidestream"); err != nil {
254267
errCount++
255268
}
269+
case "ndt":
270+
if err := CreateOrUpdateNDTWeb100(*project, "base_tables", "ndt"); err != nil {
271+
errCount++
272+
}
273+
if err := CreateOrUpdateNDTWeb100(*project, "batch", "ndt"); err != nil {
274+
errCount++
275+
}
256276

257277
case "ndt5sc":
258278
errCount += updateNDT5SC(*project)

‎schema/ndt_web100.go‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
package schema
2+
3+
import (
4+
"time"
5+
6+
"cloud.google.com/go/bigquery"
7+
"github.com/m-lab/annotation-service/api"
8+
"github.com/m-lab/go/cloud/bqx"
9+
)
10+
11+
// NDTWeb100 is a mirror struct of the BQ schema. This type is NOT USED by the parser.
12+
//
13+
// WARNING // WARNING // WARNING
14+
//
15+
// TODO: migrate parser/ndt.go to use native struct, then migrate to standard columns.
16+
type NDTWeb100 struct {
17+
TestID string `bigquery:"test_id"`
18+
TaskFilename string `bigquery:"task_filename"`
19+
ParseTime time.Time `bigquery:"parse_time"`
20+
ParserVersion string `bigquery:"parser_version"`
21+
LogTime time.Time `bigquery:"log_time"`
22+
BlacklistFlags int64 `bigquery:"blacklist_flags"`
23+
Anomalies ndtAnomalies `bigquery:"anomalies"`
24+
ConnectionSpec ndtConnectionSpec `bigquery:"connection_spec"`
25+
Web100LogEntry web100LogEntry `bigquery:"web100_log_entry"`
26+
}
27+
28+
type ndtAnomalies struct {
29+
NoMeta bool `bigquery:"no_meta"`
30+
SnaplogError bool `bigquery:"snaplog_error"`
31+
NumSnaps int64 `bigquery:"num_snaps"`
32+
BlacklistFlags int64 `bigquery:"blacklist_flags"`
33+
}
34+
35+
type ndtConnectionSpec struct {
36+
ClientAF int64 `bigquery:"client_af"`
37+
ClientApplication string `bigquery:"client_application"`
38+
ClientBrowser string `bigquery:"client_browser"`
39+
ClientHostname string `bigquery:"client_hostname"`
40+
ClientIP string `bigquery:"client_ip"`
41+
ClientKernelVersion string `bigquery:"client_kernel_version"`
42+
ClientOS string `bigquery:"client_os"`
43+
ClientVersion string `bigquery:"client_version"`
44+
DataDirection int64 `bigquery:"data_direction"`
45+
ServerAF int64 `bigquery:"server_af"`
46+
ServerHostname string `bigquery:"server_hostname"`
47+
ServerIP string `bigquery:"server_ip"`
48+
ServerKernelVersion string `bigquery:"server_kernel_version"`
49+
TLS bool `bigquery:"tls"`
50+
Websockets bool `bigquery:"websockets"`
51+
ClientGeolocation api.GeolocationIP `bigquery:"client_geolocation"`
52+
ServerGeolocation api.GeolocationIP `bigquery:"server_geolocation"`
53+
Client ndtClientNetwork `bigquery:"client"`
54+
Server ndtServerNetwork `bigquery:"server"`
55+
}
56+
57+
type network struct {
58+
ASN string `bigquery:"asn"`
59+
}
60+
61+
type ndtClientNetwork struct {
62+
Network network `bigquery:"network"`
63+
// api.ASData // Include extended asn data from annotation-service
64+
}
65+
type ndtServerNetwork struct {
66+
IataCode string `bigquery:"iata_code"`
67+
Network network `bigquery:"network"`
68+
// api.ASData // Include extended asn data from annotation-service
69+
}
70+
71+
type web100ConnectionSpec struct {
72+
LocalAF int64 `bigquery:"local_af"`
73+
LocalIP string `bigquery:"local_ip"`
74+
LocalPort int64 `bigquery:"local_port"`
75+
RemoteIP string `bigquery:"remote_ip"`
76+
RemotePort int64 `bigquery:"remote_port"`
77+
}
78+
79+
type web100LogEntry struct {
80+
LogTime int64 `bigquery:"log_time"`
81+
Version string `bigquery:"version"`
82+
ConnectionSpec web100ConnectionSpec `bigquery:"connection_spec"`
83+
Snap web100Snap `bigquery:"snap"`
84+
Deltas []web100Deltas `bigquery:"deltas"`
85+
}
86+
87+
type web100Snap struct {
88+
LocalAddress string
89+
LocalAddressType int64
90+
LocalPort int64
91+
RemAddress string
92+
RemPort int64
93+
web100SnapDelta
94+
}
95+
96+
type web100SnapDelta struct {
97+
AbruptTimeouts int64
98+
ActiveOpen int64
99+
CERcvd int64
100+
CongAvoid int64
101+
CongOverCount int64
102+
CongSignals int64
103+
CountRTT int64
104+
CurAppRQueue int64
105+
CurAppWQueue int64
106+
CurCwnd int64
107+
CurMSS int64
108+
CurRTO int64
109+
CurReasmQueue int64
110+
CurRetxQueue int64
111+
CurRwinRcvd int64
112+
CurRwinSent int64
113+
CurSsthresh int64
114+
CurTimeoutCount int64
115+
DSACKDups int64
116+
DataSegsIn int64
117+
DataSegsOut int64
118+
DupAcksIn int64
119+
DupAcksOut int64
120+
Duration int64
121+
ECN int64
122+
FastRetran int64
123+
HCDataOctetsIn int64
124+
HCDataOctetsOut int64
125+
HCThruOctetsAcked int64
126+
HCThruOctetsReceived int64
127+
LimCwnd int64
128+
LimRwin int64
129+
MSSRcvd int64
130+
MaxAppRQueue int64
131+
MaxAppWQueue int64
132+
MaxMSS int64
133+
MaxRTO int64
134+
MaxRTT int64
135+
MaxReasmQueue int64
136+
MaxRetxQueue int64
137+
MaxRwinRcvd int64
138+
MaxRwinSent int64
139+
MaxSsCwnd int64
140+
MaxSsthresh int64
141+
MinMSS int64
142+
MinRTO int64
143+
MinRTT int64
144+
MinRwinRcvd int64
145+
MinRwinSent int64
146+
MinSsthresh int64
147+
Nagle int64
148+
NonRecovDA int64
149+
OctetsRetrans int64
150+
OtherReductions int64
151+
PostCongCountRTT int64
152+
PostCongSumRTT int64
153+
PreCongSumCwnd int64
154+
PreCongSumRTT int64
155+
QuenchRcvd int64
156+
RTTVar int64
157+
RcvNxt int64
158+
RcvRTT int64
159+
RcvWindScale int64
160+
RecInitial int64
161+
RetranThresh int64
162+
SACK int64
163+
SACKBlocksRcvd int64
164+
SACKsRcvd int64
165+
SampleRTT int64
166+
SegsIn int64
167+
SegsOut int64
168+
SegsRetrans int64
169+
SendStall int64
170+
SlowStart int64
171+
SmoothedRTT int64
172+
SndInitial int64
173+
SndLimBytesCwnd int64
174+
SndLimBytesRwin int64
175+
SndLimBytesSender int64
176+
SndLimTimeCwnd int64
177+
SndLimTimeRwin int64
178+
SndLimTimeSnd int64
179+
SndLimTransCwnd int64
180+
SndLimTransRwin int64
181+
SndLimTransSnd int64
182+
SndMax int64
183+
SndNxt int64
184+
SndUna int64
185+
SndWindScale int64
186+
SpuriousFrDetected int64
187+
StartTimeStamp int64
188+
StartTimeUsec int64
189+
State int64
190+
SubsequentTimeouts int64
191+
SumRTT int64
192+
TimeStamps int64
193+
Timeouts int64
194+
WinScaleRcvd int64
195+
WinScaleSent int64
196+
X_OtherReductionsCM int64
197+
X_OtherReductionsCV int64
198+
X_Rcvbuf int64
199+
X_Sndbuf int64
200+
X_dbg1 int64
201+
X_dbg2 int64
202+
X_dbg3 int64
203+
X_dbg4 int64
204+
X_rcv_ssthresh int64
205+
X_wnd_clamp int64
206+
}
207+
208+
type web100Deltas struct {
209+
IsLast bool `bigquery:"is_last"`
210+
SnapshotNum int64 `bigquery:"snapshot_num"`
211+
DeltaIndex int64 `bigquery:"delta_index"`
212+
web100SnapDelta // embed inline struct.
213+
}
214+
215+
func (n *NDTWeb100) Schema() (bigquery.Schema, error) {
216+
sch, err := bigquery.InferSchema(n)
217+
if err != nil {
218+
return bigquery.Schema{}, err
219+
}
220+
rr := bqx.RemoveRequired(sch)
221+
return rr, nil
222+
}

0 commit comments

Comments
 (0)