Use SchemaDoc for NDTRow & Add basic generate_schema_docs CLI (#755)

* Add steps to prepare schema descriptions
* Add initial schema field description yaml files
* Add top level schema go:generate directive
* Add pre-generated file so "go get" works
* Enforce sources match generated bindata.go
* Use canonical NDTResultRow and NDTResultParser names

Author: stephen-soltesz

.travis.yml

@@ -19,6 +19,9 @@ language: go
 go:
  - 1.12
 
+env:
+- PATH=$PATH:$HOME/gopath/bin
+
 before_install:
 - sudo apt-get install -y jq  # Dependency for sync_tables_with_schema.sh.
 # Install javascript libraries
@@ -32,6 +35,9 @@ before_install:
 - go get github.com/mattn/goveralls
 - go get github.com/wadey/gocovmerge
 
+# Tool to generate and embed binary assets.
+- go get github.com/go-bindata/go-bindata/go-bindata
+
 - echo Branch is ${TRAVIS_BRANCH} and Tag is $TRAVIS_TAG
 
 # Install gcloud, for integration tests.
@@ -53,6 +59,12 @@ cache:
     - "$HOME/google-cloud-sdk/"
 
 script:
+# Enforce that the local binary assets match the generated ones.
+- cp schema/bindata.go /tmp/current-bindata.go
+- go generate ./schema
+- diff -q schema/bindata.go /tmp/current-bindata.go || (
+  echo "Files do not match; run 'go generate ./schema' and commit changes" && false )
+
 # Run all javascript tests.
 - pushd $TRAVIS_BUILD_DIR/functions
 - npm test
@@ -249,7 +261,7 @@ deploy:
     repo: m-lab/etl
     all_branches: true
     condition: $TRAVIS_BRANCH == pt-sandbox-* || $TRAVIS_BRANCH == sandbox-*
-    
+
 ## Service: etl-scamper-parser -- AppEngine Flexible Environment.
 - provider: script
   script:

cmd/generate_schema_docs/main.go

@@ -0,0 +1,126 @@
+// Copyright 2019 ETL Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//////////////////////////////////////////////////////////////////////////////
+
+// generate_schema_docs uses ETL schema field descriptions to generate
+// documentation in various formats.
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"reflect"
+	"strings"
+
+	"cloud.google.com/go/bigquery"
+	"github.com/m-lab/go/bqx"
+	"github.com/m-lab/go/flagx"
+	"github.com/m-lab/go/rtx"
+
+	"github.com/m-lab/etl/schema"
+)
+
+var usage = `
+SUMMARY
+  Format BigQuery schema field descriptions as a Markdown table.
+
+USAGE
+  $ generate_schema_docs -doc.output ./include
+  Writing include/schema_ndtresult.md
+
+`
+
+// Flags
+var (
+	outputFormat    string
+	outputDirectory string
+)
+
+func init() {
+	log.SetFlags(0)
+	flag.StringVar(&outputFormat, "doc.format", "md", "Format for output files.")
+	flag.StringVar(&outputDirectory, "doc.output", ".", "Write files to given directory.")
+
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "%s\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, usage)
+		fmt.Fprintln(os.Stderr, "Flags:")
+		flag.PrintDefaults()
+	}
+}
+
+func generateMarkdown(schema bigquery.Schema) []byte {
+	buf := &bytes.Buffer{}
+	fmt.Fprintln(buf, "| Field name       | Type       | Description    |")
+	fmt.Fprintln(buf, "| :----------------|:----------:|:---------------|")
+	bqx.WalkSchema(schema, func(prefix []string, field *bigquery.FieldSchema) error {
+		var path string
+		if len(prefix) == 1 {
+			path = ""
+		} else {
+			path = strings.Join(prefix[:len(prefix)-1], ".") + "."
+		}
+		fmt.Fprintf(buf, "| %s**%s** | %s | %s |\n", path, prefix[len(prefix)-1], field.Type, field.Description)
+		return nil
+	})
+	return buf.Bytes()
+}
+
+// All record structs define a Schema method. This interface allows us to
+// process each of them easily.
+type schemaGenerator interface {
+	Schema() (bigquery.Schema, error)
+}
+
+// shortNameOf returns the short type name of the underlying schemaGenerator type.
+// NOTE: the generator must reference an underlying pointer type,
+// e.g. `&schema.NDTResultRow{}` not `schema.NDTResultRow{}`
+func shortNameOf(g schemaGenerator) string {
+	return strings.ToLower(reflect.TypeOf(g).Elem().Name())
+}
+
+func main() {
+	flag.Parse()
+	flagx.ArgsFromEnv(flag.CommandLine)
+
+	generators := []schemaGenerator{
+		&schema.NDTResultRow{},
+		// TODO(https://github.com/m-lab/etl/issues/745): Add additional types once
+		// "standard columns" are resolved.
+	}
+
+	for _, current := range generators {
+		name := shortNameOf(current)
+		schema, err := current.Schema()
+		rtx.Must(err, "Failed to generate Schema for %s", name)
+
+		var b []byte
+		switch outputFormat {
+		case "md":
+			b = generateMarkdown(schema)
+		default:
+			log.Fatalf("Unsupported output format: %q", outputFormat)
+		}
+
+		file := path.Join(outputDirectory, "schema_"+name+"."+outputFormat)
+		log.Printf("Writing %s", file)
+		err = ioutil.WriteFile(file, b, 0644)
+		rtx.Must(err, "Failed to write file: %q", file)
+	}
+}

cmd/generate_schema_docs/main_test.go

@@ -0,0 +1,42 @@
+// Copyright 2019 ETL Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//////////////////////////////////////////////////////////////////////////////
+
+// generate_schema_docs uses ETL schema field descriptions to generate
+// documentation in various formats.
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/m-lab/go/rtx"
+)
+
+func Test_main(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "testing")
+	rtx.Must(err, "Failed to create temporary directory")
+	outputDirectory = tmpdir
+	defer os.RemoveAll(tmpdir)
+
+	main() // no crash == working
+
+	// Check for expected files in tmpdir
+	_, err = os.Stat(path.Join(tmpdir, "schema_ndtresultrow.md"))
+	if err != nil {
+		t.Errorf("main() missing output file; missing schema_ndtresultrow.md")
+	}
+}

cmd/update-schema/update.go

@@ -43,10 +43,10 @@ func CreateOrUpdatePT(project string, dataset string, table string) error {
 	return CreateOrUpdate(schema, project, dataset, table)
 }
 
-func CreateOrUpdateNDTResult(project string, dataset string, table string) error {
-	row := schema.NDTResult{}
+func CreateOrUpdateNDTResultRow(project string, dataset string, table string) error {
+	row := schema.NDTResultRow{}
 	schema, err := row.Schema()
-	rtx.Must(err, "NDTResult.Schema")
+	rtx.Must(err, "NDTResultRow.Schema")
 	return CreateOrUpdate(schema, project, dataset, table)
 }
 
@@ -119,10 +119,10 @@ func main() {
 		if err := CreateOrUpdatePT(project, "batch", "traceroute"); err != nil {
 			errCount++
 		}
-		if err := CreateOrUpdateNDTResult(project, "base_tables", "ndt5"); err != nil {
+		if err := CreateOrUpdateNDTResultRow(project, "base_tables", "ndt5"); err != nil {
 			errCount++
 		}
-		if err := CreateOrUpdateNDTResult(project, "batch", "ndt5"); err != nil {
+		if err := CreateOrUpdateNDTResultRow(project, "batch", "ndt5"); err != nil {
 			errCount++
 		}
 
@@ -143,10 +143,10 @@ func main() {
 		}
 
 	case "ndt5":
-		if err := CreateOrUpdateNDTResult(project, "base_tables", "ndt5"); err != nil {
+		if err := CreateOrUpdateNDTResultRow(project, "base_tables", "ndt5"); err != nil {
 			errCount++
 		}
-		if err := CreateOrUpdateNDTResult(project, "batch", "ndt5"); err != nil {
+		if err := CreateOrUpdateNDTResultRow(project, "batch", "ndt5"); err != nil {
 			errCount++
 		}
 

parser/ndt_result.go

@@ -45,11 +45,11 @@ func (dp *NDTResultParser) IsParsable(testName string, data []byte) (string, boo
 	return "unknown", false
 }
 
-// NOTE: NDTResult data is a JSON object that should be pushed directly into BigQuery.
+// NOTE: data.NDTResult is a JSON object that should be pushed directly into BigQuery.
 // We read the value into a struct, for compatibility with current inserter
 // backend and to eventually rely on the schema inference in m-lab/go/bqx.CreateTable().
 
-// ParseAndInsert decodes the NDT Result JSON data and inserts it into BQ.
+// ParseAndInsert decodes the data.NDTResult JSON and inserts it into BQ.
 func (dp *NDTResultParser) ParseAndInsert(meta map[string]bigquery.Value, testName string, test []byte) error {
 	// TODO: derive 'ndt5' (or 'ndt7') labels from testName.
 	metrics.WorkerState.WithLabelValues(dp.TableName(), "ndt_result").Inc()
@@ -68,7 +68,7 @@ func (dp *NDTResultParser) ParseAndInsert(meta map[string]bigquery.Value, testNa
 	rowCount := 0
 
 	for dec.More() {
-		stats := schema.NDTResult{
+		stats := schema.NDTResultRow{
 			TestID: testName,
 			ParseInfo: &schema.ParseInfo{
 				TaskFileName:  meta["filename"].(string),

parser/ndt_result_test.go

@@ -47,7 +47,7 @@ func TestNDTResultParser_ParseAndInsert(t *testing.T) {
 			if ins.Accepted() != 1 {
 				t.Fatalf("Failed to insert snaplog data.")
 			}
-			actualValues := ins.data[0].(schema.NDTResult)
+			actualValues := ins.data[0].(schema.NDTResultRow)
 			if actualValues.Result.Control == nil {
 				t.Fatal("Result.Control is nil, expected value")
 			}