Skip to content

Commit caa2a5a

Browse files
stephen-solteszstephen-soltesz
authored
Use new NDT5Result struct throughout (#850)
* Rename NDT5Result struct throughout * Rename NDT5Result description * Rename NDT5Result testdata
1 parent 5908cfd commit caa2a5a

File tree

13 files changed

+78
-78
lines changed

13 files changed

+78
-78
lines changed

‎cmd/generate_schema_docs/main.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ type schemaGenerator interface {
143143

144144
// shortNameOf returns the short type name of the underlying schemaGenerator type.
145145
// NOTE: the generator must reference an underlying pointer type,
146-
// e.g. `&schema.NDTResultRow{}` not `schema.NDTResultRow{}`
146+
// e.g. `&schema.NDT5ResultRow{}` not `schema.NDT5ResultRow{}`
147147
func shortNameOf(g schemaGenerator) string {
148148
return strings.ToLower(reflect.TypeOf(g).Elem().Name())
149149
}
@@ -153,7 +153,7 @@ func main() {
153153
flagx.ArgsFromEnv(flag.CommandLine)
154154

155155
generators := []schemaGenerator{
156-
&schema.NDTResultRow{},
156+
&schema.NDT5ResultRow{},
157157
&schema.TCPRow{},
158158
&schema.PTTest{},
159159
// TODO(https://github.com/m-lab/etl/issues/745): Add additional types once

‎cmd/update-schema/update.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@ func CreateOrUpdatePT(project string, dataset string, table string) error {
4343
return CreateOrUpdate(schema, project, dataset, table)
4444
}
4545

46-
func CreateOrUpdateNDTResultRow(project string, dataset string, table string) error {
47-
row := schema.NDTResultRow{}
46+
func CreateOrUpdateNDT5ResultRow(project string, dataset string, table string) error {
47+
row := schema.NDT5ResultRow{}
4848
schema, err := row.Schema()
49-
rtx.Must(err, "NDTResultRow.Schema")
49+
rtx.Must(err, "NDT5ResultRow.Schema")
5050
return CreateOrUpdate(schema, project, dataset, table)
5151
}
5252

@@ -120,10 +120,10 @@ func main() {
120120
errCount++
121121
}
122122
/* Temporaily comment out due to NDT7 unstable schemas.
123-
if err := CreateOrUpdateNDTResultRow(project, "base_tables", "ndt5"); err != nil {
123+
if err := CreateOrUpdateNDT5ResultRow(project, "base_tables", "ndt5"); err != nil {
124124
errCount++
125125
}
126-
if err := CreateOrUpdateNDTResultRow(project, "batch", "ndt5"); err != nil {
126+
if err := CreateOrUpdateNDT5ResultRow(project, "batch", "ndt5"); err != nil {
127127
errCount++
128128
}*/
129129

@@ -144,10 +144,10 @@ func main() {
144144
}
145145

146146
case "ndt5":
147-
if err := CreateOrUpdateNDTResultRow(project, "base_tables", "ndt5"); err != nil {
147+
if err := CreateOrUpdateNDT5ResultRow(project, "base_tables", "ndt5"); err != nil {
148148
errCount++
149149
}
150-
if err := CreateOrUpdateNDTResultRow(project, "batch", "ndt5"); err != nil {
150+
if err := CreateOrUpdateNDT5ResultRow(project, "batch", "ndt5"); err != nil {
151151
errCount++
152152
}
153153

‎parser/ndt_result.go‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package parser
22

3-
// This file defines the Parser subtype that handles NDTResult data.
3+
// This file defines the Parser subtype that handles NDT5Result data.
44

55
import (
66
"bytes"
@@ -21,30 +21,30 @@ import (
2121
)
2222

2323
//=====================================================================================
24-
// NDTResult Parser
24+
// NDT5Result Parser
2525
//=====================================================================================
2626

27-
// NDTResultParser
28-
type NDTResultParser struct {
27+
// NDT5ResultParser
28+
type NDT5ResultParser struct {
2929
*row.Base
3030
table string
3131
suffix string
3232
}
3333

34-
func NewNDTResultParser(sink row.Sink, table, suffix string, ann v2as.Annotator) etl.Parser {
34+
func NewNDT5ResultParser(sink row.Sink, table, suffix string, ann v2as.Annotator) etl.Parser {
3535
bufSize := etl.NDT5.BQBufferSize()
3636
if ann == nil {
3737
ann = v2as.GetAnnotator(annotation.BatchURL)
3838
}
3939

40-
return &NDTResultParser{
40+
return &NDT5ResultParser{
4141
Base: row.NewBase("foobar", sink, bufSize, ann),
4242
table: table,
4343
suffix: suffix,
4444
}
4545
}
4646

47-
func (dp *NDTResultParser) TaskError() error {
47+
func (dp *NDT5ResultParser) TaskError() error {
4848
stats := dp.GetStats()
4949
if stats.Total() < 10*stats.Failed {
5050
log.Printf("Warning: high row insert errors (more than 10%%): %d failed of %d accepted\n",
@@ -55,20 +55,20 @@ func (dp *NDTResultParser) TaskError() error {
5555
}
5656

5757
// IsParsable returns the canonical test type and whether to parse data.
58-
func (dp *NDTResultParser) IsParsable(testName string, data []byte) (string, bool) {
58+
func (dp *NDT5ResultParser) IsParsable(testName string, data []byte) (string, bool) {
5959
// Files look like: "<UUID>.json"
6060
if strings.HasSuffix(testName, "json") {
6161
return "ndt_result", true
6262
}
6363
return "unknown", false
6464
}
6565

66-
// NOTE: data.NDTResult is a JSON object that should be pushed directly into BigQuery.
66+
// NOTE: data.NDT5Result is a JSON object that should be pushed directly into BigQuery.
6767
// We read the value into a struct, for compatibility with current inserter
6868
// backend and to eventually rely on the schema inference in m-lab/go/bqx.CreateTable().
6969

70-
// ParseAndInsert decodes the data.NDTResult JSON and inserts it into BQ.
71-
func (dp *NDTResultParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, test []byte) error {
70+
// ParseAndInsert decodes the data.NDT5Result JSON and inserts it into BQ.
71+
func (dp *NDT5ResultParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, test []byte) error {
7272
// TODO: derive 'ndt5' (or 'ndt7') labels from testName.
7373
metrics.WorkerState.WithLabelValues(dp.TableName(), "ndt_result").Inc()
7474
defer metrics.WorkerState.WithLabelValues(dp.TableName(), "ndt_result").Dec()
@@ -85,7 +85,7 @@ func (dp *NDTResultParser) ParseAndInsert(meta map[string]bigquery.Value, testNa
8585
dec := json.NewDecoder(rdr)
8686

8787
for dec.More() {
88-
stats := schema.NDTResultRow{
88+
stats := schema.NDT5ResultRow{
8989
TestID: testName,
9090
ParseInfo: &schema.ParseInfo{
9191
TaskFileName: meta["filename"].(string),
@@ -117,36 +117,36 @@ func (dp *NDTResultParser) ParseAndInsert(meta map[string]bigquery.Value, testNa
117117
}
118118

119119
// NB: These functions are also required to complete the etl.Parser interface.
120-
// For NDTResult, we just forward the calls to the Inserter.
120+
// For NDT5Result, we just forward the calls to the Inserter.
121121

122-
func (dp *NDTResultParser) Flush() error {
122+
func (dp *NDT5ResultParser) Flush() error {
123123
return dp.Base.Flush()
124124
}
125125

126-
func (dp *NDTResultParser) TableName() string {
126+
func (dp *NDT5ResultParser) TableName() string {
127127
return dp.table
128128
}
129129

130-
func (dp *NDTResultParser) FullTableName() string {
130+
func (dp *NDT5ResultParser) FullTableName() string {
131131
return dp.table + dp.suffix
132132
}
133133

134134
// RowsInBuffer returns the count of rows currently in the buffer.
135-
func (dp *NDTResultParser) RowsInBuffer() int {
135+
func (dp *NDT5ResultParser) RowsInBuffer() int {
136136
return dp.GetStats().Pending
137137
}
138138

139139
// Committed returns the count of rows successfully committed to BQ.
140-
func (dp *NDTResultParser) Committed() int {
140+
func (dp *NDT5ResultParser) Committed() int {
141141
return dp.GetStats().Committed
142142
}
143143

144144
// Accepted returns the count of all rows received through InsertRow(s)
145-
func (dp *NDTResultParser) Accepted() int {
145+
func (dp *NDT5ResultParser) Accepted() int {
146146
return dp.GetStats().Total()
147147
}
148148

149149
// Failed returns the count of all rows that could not be committed.
150-
func (dp *NDTResultParser) Failed() int {
150+
func (dp *NDT5ResultParser) Failed() int {
151151
return dp.GetStats().Failed
152152
}

‎parser/ndt_result_test.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/m-lab/etl/schema"
1111
)
1212

13-
func TestNDTResultParser_ParseAndInsert(t *testing.T) {
13+
func TestNDT5ResultParser_ParseAndInsert(t *testing.T) {
1414
tests := []struct {
1515
name string
1616
testName string
@@ -31,9 +31,9 @@ func TestNDTResultParser_ParseAndInsert(t *testing.T) {
3131
for _, tt := range tests {
3232
t.Run(tt.name, func(t *testing.T) {
3333
ins := newInMemorySink()
34-
n := parser.NewNDTResultParser(ins, "test", "_suffix", nil)
34+
n := parser.NewNDT5ResultParser(ins, "test", "_suffix", nil)
3535

36-
resultData, err := ioutil.ReadFile(`testdata/NDTResult/` + tt.testName)
36+
resultData, err := ioutil.ReadFile(`testdata/NDT5Result/` + tt.testName)
3737
if err != nil {
3838
t.Fatalf(err.Error())
3939
}
@@ -42,13 +42,13 @@ func TestNDTResultParser_ParseAndInsert(t *testing.T) {
4242
}
4343

4444
if err := n.ParseAndInsert(meta, tt.testName, resultData); (err != nil) != tt.wantErr {
45-
t.Errorf("NDTResultParser.ParseAndInsert() error = %v, wantErr %v", err, tt.wantErr)
45+
t.Errorf("NDT5ResultParser.ParseAndInsert() error = %v, wantErr %v", err, tt.wantErr)
4646
}
4747
if n.Accepted() != 1 {
4848
t.Fatal("Failed to insert snaplog data.", ins)
4949
}
5050
n.Flush()
51-
actualValues := ins.data[0].(*schema.NDTResultRow)
51+
actualValues := ins.data[0].(*schema.NDT5ResultRow)
5252
if actualValues.Result.Control == nil {
5353
t.Fatal("Result.Control is nil, expected value")
5454
}
@@ -67,7 +67,7 @@ func TestNDTResultParser_ParseAndInsert(t *testing.T) {
6767
}
6868
}
6969

70-
func TestNDTResultParser_IsParsable(t *testing.T) {
70+
func TestNDT5ResultParser_IsParsable(t *testing.T) {
7171
tests := []struct {
7272
name string
7373
testName string
@@ -86,14 +86,14 @@ func TestNDTResultParser_IsParsable(t *testing.T) {
8686
}
8787
for _, tt := range tests {
8888
t.Run(tt.name, func(t *testing.T) {
89-
data, err := ioutil.ReadFile(`testdata/NDTResult/` + tt.testName)
89+
data, err := ioutil.ReadFile(`testdata/NDT5Result/` + tt.testName)
9090
if err != nil {
9191
t.Fatalf(err.Error())
9292
}
93-
p := &parser.NDTResultParser{}
93+
p := &parser.NDT5ResultParser{}
9494
_, got := p.IsParsable(tt.testName, data)
9595
if got != tt.want {
96-
t.Errorf("NDTResultParser.IsParsable() got1 = %v, want %v", got, tt.want)
96+
t.Errorf("NDT5ResultParser.IsParsable() got1 = %v, want %v", got, tt.want)
9797
}
9898
})
9999
}

‎parser/parser.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func NewParser(dt etl.DataType, ins etl.Inserter) etl.Parser {
5353
log.Println(reflect.TypeOf(ins))
5454
return nil
5555
}
56-
return NewNDTResultParser(sink, ins.TableBase(), ins.TableSuffix(), nil)
56+
return NewNDT5ResultParser(sink, ins.TableBase(), ins.TableSuffix(), nil)
5757

5858
case etl.SS:
5959
return NewDefaultSSParser(ins) // TODO fix this hack.

parser/testdata/NDTResult/badfile.badextension renamed to parser/testdata/NDT5Result/badfile.badextension

File renamed without changes.

parser/testdata/NDTResult/ndt-5hkck_1566219987_000000000000017D.json renamed to parser/testdata/NDT5Result/ndt-5hkck_1566219987_000000000000017D.json

File renamed without changes.

parser/testdata/NDTResult/ndt-vscqp_1565987984_000000000001A1C2.json renamed to parser/testdata/NDT5Result/ndt-vscqp_1565987984_000000000001A1C2.json

File renamed without changes.

‎schema/bindata.go‎

Lines changed: 18 additions & 18 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

schema/descriptions/NDTResultRow.yaml renamed to schema/descriptions/NDT5ResultRow.yaml

File renamed without changes.

0 commit comments

Comments
 (0)