|
| 1 | +package parser_test |
| 2 | + |
| 3 | +import ( |
| 4 | + "log" |
| 5 | + "testing" |
| 6 | + "time" |
| 7 | + |
| 8 | + "cloud.google.com/go/bigquery" |
| 9 | + |
| 10 | + "github.com/m-lab/etl/bq" |
| 11 | + "github.com/m-lab/etl/etl" |
| 12 | + "github.com/m-lab/etl/fake" |
| 13 | + "github.com/m-lab/etl/parser" |
| 14 | +) |
| 15 | + |
| 16 | +func init() { |
| 17 | + log.SetFlags(log.LstdFlags | log.Lshortfile) |
| 18 | +} |
| 19 | + |
| 20 | +var test_data []byte = []byte( |
| 21 | + `{ |
| 22 | + "sample": [{"timestamp": 69850, "value": 0.0}, {"timestamp": 69860, "value": 0.0}], |
| 23 | + "metric": "switch.multicast.local.rx", |
| 24 | + "hostname": "mlab4.sea05.measurement-lab.org", |
| 25 | + "experiment": "s1.sea05.measurement-lab.org"} |
| 26 | + {"sample": [], |
| 27 | + "metric": "switch.multicast.local.rx", |
| 28 | + "hostname": "mlab1.sea05.measurement-lab.org", |
| 29 | + "experiment": "s1.sea05.measurement-lab.org"}`) |
| 30 | + |
| 31 | +// This tests the parser, using a fake inserter, so that it runs entirely locally. |
| 32 | +func TestJSONParsing(t *testing.T) { |
| 33 | + // This creates a real inserter, with a fake uploader, for local testing. |
| 34 | + uploader := fake.FakeUploader{} |
| 35 | + ins, err := bq.NewBQInserter(etl.InserterParams{ |
| 36 | + Project: "mlab-sandbox", Dataset: "dataset", Table: "disco_test", Suffix: "", |
| 37 | + BufferSize: 3, PutTimeout: 10 * time.Second, MaxRetryDelay: time.Second}, &uploader) |
| 38 | + if err != nil { |
| 39 | + t.Fatal(err) |
| 40 | + } |
| 41 | + |
| 42 | + var parser etl.Parser = parser.NewDiscoParser(ins) |
| 43 | + |
| 44 | + meta := map[string]bigquery.Value{"filename": "fake-filename.tar", "parse_time": time.Now()} |
| 45 | + // Should result in two tests sent to inserter, but no call to uploader. |
| 46 | + err = parser.ParseAndInsert(meta, "testName", test_data) |
| 47 | + if err != nil { |
| 48 | + t.Fatal(err) |
| 49 | + } |
| 50 | + if ins.Accepted() != 2 { |
| 51 | + t.Error("Accepted = ", ins.Accepted()) |
| 52 | + t.Fail() |
| 53 | + } |
| 54 | + |
| 55 | + // Adds two more rows, triggering an upload of 3 rows. |
| 56 | + err = parser.ParseAndInsert(meta, "testName", test_data) |
| 57 | + if err != nil { |
| 58 | + t.Fatal(err) |
| 59 | + } |
| 60 | + if len(uploader.Rows) != 3 { |
| 61 | + t.Error("Expected 3, got", len(uploader.Rows)) |
| 62 | + } |
| 63 | + // The testName was that of a DISCOv1 filename, for which the parser omits |
| 64 | + // the last sample for each metric, so even though there are two input |
| 65 | + // samples there should only be one in the resulting row. |
| 66 | + if uploader.Rows[1].Row["sample"] != nil && len(uploader.Rows[1].Row["sample"].([]bigquery.Value)) != 1 { |
| 67 | + t.Error("Expected 1, got", len(uploader.Rows[1].Row["sample"].([]bigquery.Value))) |
| 68 | + } |
| 69 | + |
| 70 | + // Adds two more rows, triggering an upload of 3 rows. |
| 71 | + err = parser.ParseAndInsert(meta, "testName-switch.jsonl", test_data) |
| 72 | + if err != nil { |
| 73 | + t.Fatal(err) |
| 74 | + } |
| 75 | + |
| 76 | + if ins.Accepted() != 6 { |
| 77 | + t.Error("Accepted = ", ins.Accepted()) |
| 78 | + } |
| 79 | + if ins.RowsInBuffer() != 0 { |
| 80 | + t.Error("RowsInBuffer = ", ins.RowsInBuffer()) |
| 81 | + } |
| 82 | + if len(uploader.Rows) != 3 { |
| 83 | + t.Error("Expected 3, got", len(uploader.Rows)) |
| 84 | + } |
| 85 | + |
| 86 | + if uploader.Rows[0].Row["sample"] != nil && len(uploader.Rows[0].Row["sample"].([]bigquery.Value)) != 1 { |
| 87 | + t.Error("Expected 1, got", len(uploader.Rows[0].Row["sample"].([]bigquery.Value))) |
| 88 | + } |
| 89 | + // The testName was that of a DISCOv2 filename (suffix of -switch.jsonl), |
| 90 | + // for which the parser should include all samples. Therefore, since the |
| 91 | + // input had two samples, so should the resulting row. |
| 92 | + if uploader.Rows[1].Row["sample"] != nil && len(uploader.Rows[1].Row["sample"].([]bigquery.Value)) != 2 { |
| 93 | + t.Error("Expected 2, got", len(uploader.Rows[1].Row["sample"].([]bigquery.Value))) |
| 94 | + } |
| 95 | + if uploader.Rows[0].Row["task_filename"].(string) != "fake-filename.tar" { |
| 96 | + t.Error("task_filename incorrect: Expected 'fake-filename.tar', got", |
| 97 | + uploader.Rows[0].Row["task_filename"].(string)) |
| 98 | + } |
| 99 | + if uploader.Rows[0].Row["test_id"].(string) != "testName" { |
| 100 | + t.Error("task_filename incorrect: Expected 'testName', got", |
| 101 | + uploader.Rows[0].Row["test_id"].(string)) |
| 102 | + } |
| 103 | + if uploader.Rows[0].Row["metric"].(string) != "switch.multicast.local.rx" { |
| 104 | + t.Error("task_filename incorrect: Expected 'switch.multicast.local.rx', got", |
| 105 | + uploader.Rows[0].Row["metric"].(string)) |
| 106 | + } |
| 107 | + if uploader.Rows[0].Row["hostname"].(string) != "mlab1.sea05.measurement-lab.org" { |
| 108 | + t.Error("task_filename incorrect: Expected 'mlab1.sea05.measuremet-lab.org', got", |
| 109 | + uploader.Rows[0].Row["hostname"].(string)) |
| 110 | + } |
| 111 | + if uploader.Rows[0].Row["experiment"].(string) != "s1.sea05.measurement-lab.org" { |
| 112 | + t.Error("task_filename incorrect: Expected 's1.sea05.measuremet-lab.org', got", |
| 113 | + uploader.Rows[0].Row["experiment"].(string)) |
| 114 | + } |
| 115 | + |
| 116 | + if err != nil { |
| 117 | + log.Printf("Request: %v\n", uploader.Request) |
| 118 | + log.Printf("Rows Len: %d\n", len(uploader.Rows)) |
| 119 | + if len(uploader.Rows) > 0 { |
| 120 | + log.Printf("Rows[0]: %v\n", uploader.Rows[0]) |
| 121 | + log.Printf("Rows[0]['sample']: %v\n", len(uploader.Rows[0].Row["sample"].([]bigquery.Value))) |
| 122 | + } |
| 123 | + t.Error(err) |
| 124 | + } |
| 125 | +} |
| 126 | + |
| 127 | +// DISABLED |
| 128 | +// This tests insertion into a test table in the cloud. Should not normally be executed. |
| 129 | +func xTestRealBackend(t *testing.T) { |
| 130 | + ins, err := bq.NewInserter(etl.SW, time.Now()) |
| 131 | + var parser etl.Parser = parser.NewDiscoParser(ins) |
| 132 | + |
| 133 | + meta := map[string]bigquery.Value{"filename": "filename", "parse_time": time.Now()} |
| 134 | + for i := 0; i < 3; i++ { |
| 135 | + // Iterations: |
| 136 | + // Add two rows, no upload. |
| 137 | + // Add two more rows, triggering an upload of 3 rows. |
| 138 | + // Add two more rows, triggering an upload of 3 rows. |
| 139 | + err = parser.ParseAndInsert(meta, "testName", test_data) |
| 140 | + if ins.Accepted() != 2 { |
| 141 | + t.Error("Accepted = ", ins.Accepted()) |
| 142 | + t.Fail() |
| 143 | + } |
| 144 | + } |
| 145 | + |
| 146 | + if ins.Accepted() != 6 { |
| 147 | + t.Error("Accepted = ", ins.Accepted()) |
| 148 | + } |
| 149 | + if ins.RowsInBuffer() != 0 { |
| 150 | + t.Error("RowsInBuffer = ", ins.RowsInBuffer()) |
| 151 | + } |
| 152 | + |
| 153 | + if err != nil { |
| 154 | + t.Error(err) |
| 155 | + } |
| 156 | +} |
0 commit comments