Skip to content

Commit 56a2669

Browse files
cristinaleonrcristinaleonr
authored
Rename etl_test_count and update it for unsuccessful scamper parsing (#1053)
* Filter out legacy errors from scamper1 parsing errors * Return ErrIsInvalid * Use custom error * Fix failed stats * Revert "Fix failed stats" This reverts commit 062f349. * Use TestCount metric * Change to etl_test_total * Rename in worker_test.go * Rename in worker_test.go fix * Remove error filtering * Add TestTotal to error
1 parent 6347d8b commit 56a2669

File tree

21 files changed

+80
-59
lines changed

21 files changed

+80
-59
lines changed

‎metrics/metrics.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -195,16 +195,16 @@ var (
195195
[]string{"table", "status"},
196196
)
197197

198-
// TestCount counts the number of tests successfully processed by the parsers.
198+
// TestTotal counts the number of tests successfully processed by the parsers.
199199
//
200200
// Provides metrics:
201-
// etl_test_count{table, filetype, status}
201+
// etl_test_total{table, filetype, status}
202202
// Example usage:
203-
// metrics.TestCount.WithLabelValues(
203+
// metrics.TestTotal.WithLabelValues(
204204
// tt.Inserter.TableBase(), "s2c", "ok").Inc()
205-
TestCount = promauto.NewCounterVec(
205+
TestTotal = promauto.NewCounterVec(
206206
prometheus.CounterOpts{
207-
Name: "etl_test_count",
207+
Name: "etl_test_total",
208208
Help: "Number of tests processed.",
209209
},
210210
// ndt/pt/ss, s2c/c2s/meta, ok/reject/error/

‎metrics/metrics_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ func TestMetrics(t *testing.T) {
100100
metrics.PTTestCount.WithLabelValues("x")
101101
metrics.RowSizeHistogram.WithLabelValues("x")
102102
metrics.TaskTotal.WithLabelValues("x", "x")
103-
metrics.TestCount.WithLabelValues("x", "x", "x")
103+
metrics.TestTotal.WithLabelValues("x", "x", "x")
104104
metrics.WarningCount.WithLabelValues("x", "x", "x")
105105
metrics.WorkerCount.WithLabelValues("x")
106106
metrics.WorkerState.WithLabelValues("x", "x")

‎parser/annotation.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func (ap *AnnotationParser) ParseAndInsert(meta map[string]bigquery.Value, testN
9292
err := json.Unmarshal(test, &raw)
9393
if err != nil {
9494
log.Println(err)
95-
metrics.TestCount.WithLabelValues(ap.TableName(), "annotation", "decode-location-error").Inc()
95+
metrics.TestTotal.WithLabelValues(ap.TableName(), "annotation", "decode-location-error").Inc()
9696
return err
9797
}
9898

@@ -118,7 +118,7 @@ func (ap *AnnotationParser) ParseAndInsert(meta map[string]bigquery.Value, testN
118118
}
119119

120120
// Count successful inserts.
121-
metrics.TestCount.WithLabelValues(ap.TableName(), "annotation", "ok").Inc()
121+
metrics.TestTotal.WithLabelValues(ap.TableName(), "annotation", "ok").Inc()
122122
return nil
123123
}
124124

‎parser/disco.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ func (dp *DiscoParser) ParseAndInsert(meta map[string]bigquery.Value, testName s
8181
tmp := schema.SwitchStats{}
8282
err := dec.Decode(&tmp)
8383
if err != nil {
84-
metrics.TestCount.WithLabelValues(
84+
metrics.TestTotal.WithLabelValues(
8585
dp.TableName(), "disco", "Decode").Inc()
8686
// TODO(dev) Should accumulate errors, instead of aborting?
8787
return err
@@ -133,18 +133,18 @@ func (dp *DiscoParser) ParseAndInsert(meta map[string]bigquery.Value, testName s
133133
switch t := err.(type) {
134134
case bigquery.PutMultiError:
135135
// TODO improve error handling??
136-
metrics.TestCount.WithLabelValues(
136+
metrics.TestTotal.WithLabelValues(
137137
dp.TableName(), "disco", "insert-multi").Inc()
138138
log.Printf("%v\n", t[0].Error())
139139
default:
140-
metrics.TestCount.WithLabelValues(
140+
metrics.TestTotal.WithLabelValues(
141141
dp.TableName(), "disco", "insert-other").Inc()
142142
}
143143
// TODO(dev) Should accumulate errors, instead of aborting?
144144
return err
145145
}
146146
// Count successful inserts.
147-
metrics.TestCount.WithLabelValues(dp.TableName(), "disco", "ok").Inc()
147+
metrics.TestTotal.WithLabelValues(dp.TableName(), "disco", "ok").Inc()
148148
}
149149

150150
// Measure the distribution of records per file.

‎parser/hopannotation1.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ func (p *HopAnnotation1Parser) ParseAndInsert(fileMetadata map[string]bigquery.V
6666
raw := hopannotation.HopAnnotation1{}
6767
err := json.Unmarshal(rawContent, &raw)
6868
if err != nil {
69-
metrics.TestCount.WithLabelValues(p.TableName(), "hopannotation1", "decode-location-error").Inc()
69+
metrics.TestTotal.WithLabelValues(p.TableName(), "hopannotation1", "decode-location-error").Inc()
7070
return err
7171
}
7272

@@ -88,7 +88,7 @@ func (p *HopAnnotation1Parser) ParseAndInsert(fileMetadata map[string]bigquery.V
8888
return err
8989
}
9090
// Count successful inserts.
91-
metrics.TestCount.WithLabelValues(p.TableName(), "hopannotation1", "ok").Inc()
91+
metrics.TestTotal.WithLabelValues(p.TableName(), "hopannotation1", "ok").Inc()
9292

9393
return nil
9494
}

‎parser/ndt.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ func (n *NDTParser) IsParsable(testName string, data []byte) (string, bool) {
210210
return info.Suffix, false
211211
}
212212
// All other cases.
213-
metrics.TestCount.WithLabelValues(
213+
metrics.TestTotal.WithLabelValues(
214214
n.TableName(), "unknown", "unknown suffix").Inc()
215215
return "unknown", false
216216
}
@@ -226,7 +226,7 @@ func (n *NDTParser) ParseAndInsert(taskInfo map[string]bigquery.Value, testName
226226
// TODO(prod) Ensure that archive files are also date sorted.
227227
info, err := ParseNDTFileName(testName)
228228
if err != nil {
229-
metrics.TestCount.WithLabelValues(
229+
metrics.TestTotal.WithLabelValues(
230230
n.TableName(), "unknown", "bad filename").Inc()
231231
log.Println(err)
232232
return nil
@@ -251,7 +251,7 @@ func (n *NDTParser) ParseAndInsert(taskInfo map[string]bigquery.Value, testName
251251
} else {
252252
// Within a group of tests, we expect consistent taskInfo.
253253
if n.taskFileName != taskInfo["filename"].(string) {
254-
metrics.TestCount.WithLabelValues(
254+
metrics.TestTotal.WithLabelValues(
255255
n.TableName(), "any", "inconsistent taskFileName").Inc()
256256
}
257257
}
@@ -309,7 +309,7 @@ func (n *NDTParser) ParseAndInsert(taskInfo map[string]bigquery.Value, testName
309309
n.metaFile = ProcessMetaFile(
310310
n.TableName(), n.TableSuffix(), testName, content)
311311
default:
312-
metrics.TestCount.WithLabelValues(
312+
metrics.TestTotal.WithLabelValues(
313313
n.TableName(), "unknown", "unparsable file").Inc()
314314
return errors.New("Unknown test suffix: " + info.Suffix)
315315
}
@@ -407,7 +407,7 @@ func (n *NDTParser) getDeltas(snaplog *web100.SnapLog, testType string) ([]schem
407407
snap, err := snaplog.Snapshot(count)
408408
if err != nil {
409409
// TODO - refine label and maybe write a log?
410-
metrics.TestCount.WithLabelValues(
410+
metrics.TestTotal.WithLabelValues(
411411
n.TableName(), testType, "snapshot failure").Inc()
412412
return nil, 0
413413
}
@@ -506,7 +506,7 @@ func (n *NDTParser) getAndInsertValues(test *fileInfoAndData, testType string) {
506506
if err != nil {
507507
metrics.ErrorCount.WithLabelValues(
508508
n.TableName(), testType, "final snapshot failure").Inc()
509-
metrics.TestCount.WithLabelValues(
509+
metrics.TestTotal.WithLabelValues(
510510
n.TableName(), testType, "final snapshot failure").Inc()
511511
return
512512
}
@@ -515,7 +515,7 @@ func (n *NDTParser) getAndInsertValues(test *fileInfoAndData, testType string) {
515515
if err != nil {
516516
metrics.ErrorCount.WithLabelValues(
517517
n.TableName(), testType, "final snapValues failure").Inc()
518-
metrics.TestCount.WithLabelValues(
518+
metrics.TestTotal.WithLabelValues(
519519
n.TableName(), testType, "final snapValues failure").Inc()
520520
log.Printf("Error calling SnapshotValues() in test %s, when processing: %s\n%s\n",
521521
test.fn, n.taskFileName, err)
@@ -633,7 +633,7 @@ func (n *NDTParser) getAndInsertValues(test *fileInfoAndData, testType string) {
633633
return
634634
}
635635

636-
metrics.TestCount.WithLabelValues(
636+
metrics.TestTotal.WithLabelValues(
637637
n.TableName(), testType, "ok").Inc()
638638
}
639639

‎parser/ndt5_result.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ func (dp *NDT5ResultParser) ParseAndInsert(meta map[string]bigquery.Value, testN
100100
err := dec.Decode(&stats.Result)
101101
if err != nil {
102102
log.Println(err)
103-
metrics.TestCount.WithLabelValues(
103+
metrics.TestTotal.WithLabelValues(
104104
dp.TableName(), "ndt5_result", "Decode").Inc()
105105
return err
106106
}
@@ -116,7 +116,7 @@ func (dp *NDT5ResultParser) ParseAndInsert(meta map[string]bigquery.Value, testN
116116
return err
117117
}
118118
// Count successful inserts.
119-
metrics.TestCount.WithLabelValues(dp.TableName(), "ndt5_result", "ok").Inc()
119+
metrics.TestTotal.WithLabelValues(dp.TableName(), "ndt5_result", "ok").Inc()
120120
}
121121

122122
return nil

‎parser/ndt7_result.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ func (dp *NDT7ResultParser) ParseAndInsert(meta map[string]bigquery.Value, testN
8989
err := json.Unmarshal(test, &row.Raw)
9090
if err != nil {
9191
log.Println(meta["filename"].(string), testName, err)
92-
metrics.TestCount.WithLabelValues(dp.TableName(), "ndt7_result", "Unmarshal").Inc()
92+
metrics.TestTotal.WithLabelValues(dp.TableName(), "ndt7_result", "Unmarshal").Inc()
9393
return err
9494
}
9595

@@ -132,7 +132,7 @@ func (dp *NDT7ResultParser) ParseAndInsert(meta map[string]bigquery.Value, testN
132132
return err
133133
}
134134
// Count successful inserts.
135-
metrics.TestCount.WithLabelValues(dp.TableName(), "ndt7_result", "ok").Inc()
135+
metrics.TestTotal.WithLabelValues(dp.TableName(), "ndt7_result", "ok").Inc()
136136
return nil
137137
}
138138

‎parser/ndt_meta.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -198,20 +198,20 @@ func ProcessMetaFile(tableName string, suffix string, testName string, content [
198198
// Create a map from the metafile raw content
199199
metamap, err := parseMetaFile(content)
200200
if err != nil {
201-
metrics.TestCount.WithLabelValues(
201+
metrics.TestTotal.WithLabelValues(
202202
tableName, "meta", "error").Inc()
203203
log.Println("meta processing error: " + err.Error())
204204
return nil
205205
}
206206
metaFile, err := createMetaFileData(testName, metamap)
207207
if err != nil {
208-
metrics.TestCount.WithLabelValues(
208+
metrics.TestTotal.WithLabelValues(
209209
tableName, "meta", "error").Inc()
210210
log.Println("meta processing error: " + err.Error())
211211
return nil
212212
}
213213

214-
metrics.TestCount.WithLabelValues(
214+
metrics.TestTotal.WithLabelValues(
215215
tableName, "meta", "ok").Inc()
216216
return metaFile
217217
}

‎parser/parser.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ type NullParser struct {
173173
}
174174

175175
func (np *NullParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, test []byte) error {
176-
metrics.TestCount.WithLabelValues("table", "null", "ok").Inc()
176+
metrics.TestTotal.WithLabelValues("table", "null", "ok").Inc()
177177
return nil
178178
}
179179
func (np *NullParser) TableName() string {

0 commit comments

Comments
 (0)