Automate the schema change in travis (#815)

yachang · web-flow · commit 18a6d5419a9e · 2020-02-12T14:42:46.000-05:00
* update submodule to latest master

* Update .travis.yml

* Update .travis.yml

* minor

* add +x

* update script permission.

* trial to get rid of "local"

* more

* disable ndt5 table schema update
diff --git a/.travis.yml b/.travis.yml
@@ -132,6 +132,8 @@ script:
 # Clean build and prepare for deployment
 - cd $TRAVIS_BUILD_DIR/cmd/etl_worker &&
   go build -v -ldflags "-X github.com/m-lab/go/prometheusx.GitShortCommit=$(git log -1 --format=%h)" .
+- cd $TRAVIS_BUILD_DIR/cmd/update-schema &&
+  go build -v
 - cd $TRAVIS_BUILD_DIR
 
 #################################################################################
@@ -204,6 +206,8 @@ deploy:
     $TRAVIS_BUILD_DIR/travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_sandbox
     && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-sandbox
     SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/etl_worker app-batch.yaml
+    && $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-sandbox
+    SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
 #    && $TRAVIS_BUILD_DIR/etl-schema/schema/sync_tables_with_schema.sh mlab-sandbox batch nodryrun
 #    && $TRAVIS_BUILD_DIR/etl-schema/schema/sync_tables_with_schema.sh mlab-sandbox base_tables nodryrun
   skip_cleanup: true
@@ -216,6 +220,8 @@ deploy:
 - provider: script
   script:
     $TRAVIS_BUILD_DIR/travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_sandbox
+    && $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-sandbox
+    SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
 #   && $TRAVIS_BUILD_DIR/etl-schema/schema/sync_tables_with_schema.sh mlab-sandbox base_tables nodryrun
   skip_cleanup: true
   on:
@@ -233,6 +239,8 @@ deploy:
 - provider: script
   script:
     $TRAVIS_BUILD_DIR/travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_sandbox
+    && $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-sandbox
+    SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
     && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-sandbox
     SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/etl_worker app-ndt.yaml
   skip_cleanup: true
@@ -269,7 +277,9 @@ deploy:
 ## Service: etl-traceroute-parser -- AppEngine Flexible Environment.
 - provider: script
   script:
-    $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-sandbox
+    $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-sandbox
+    SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
+    && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-sandbox
     SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/etl_worker app-traceroute.yaml
   skip_cleanup: true
   on:
@@ -280,7 +290,9 @@ deploy:
 ## Service: etl-scamper-parser -- AppEngine Flexible Environment.
 - provider: script
   script:
-    $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-sandbox
+    $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-sandbox
+    SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
+    && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-sandbox
     SERVICE_ACCOUNT_mlab_sandbox $TRAVIS_BUILD_DIR/cmd/etl_worker app-scamper.yaml
   skip_cleanup: true
   on:
@@ -342,6 +354,9 @@ deploy:
     &&
     travis/kubectl.sh mlab-staging data-processing ./apply-cluster.sh
     &&
+    $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-staging
+    SERVICE_ACCOUNT_mlab_staging $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
+    &&
     $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-staging
     SERVICE_ACCOUNT_mlab_staging $TRAVIS_BUILD_DIR/cmd/etl_worker app-ndt.yaml
     && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-staging
@@ -391,6 +406,8 @@ deploy:
 - provider: script
   script:
     $TRAVIS_BUILD_DIR/travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_oti
+    && $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-oti
+    SERVICE_ACCOUNT_mlab_oti $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
     && cd $TRAVIS_BUILD_DIR/functions
     && gcloud functions deploy createProdTaskOnFileNotification --project=mlab-oti --stage-bucket=functions-mlab-oti --trigger-event=providers/cloud.storage/eventTypes/object.change --trigger-resource=archive-mlab-oti
     && gcloud functions deploy createProdTaskOnEmbargoFileNotification --project=mlab-oti --stage-bucket=functions-mlab-oti --trigger-event=providers/cloud.storage/eventTypes/object.change --trigger-resource=embargo-mlab-oti
@@ -413,6 +430,8 @@ deploy:
 - provider: script
   script:
     $TRAVIS_BUILD_DIR/travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_oti
+    && $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-oti
+    SERVICE_ACCOUNT_mlab_oti $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
     && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-oti
     SERVICE_ACCOUNT_mlab_oti $TRAVIS_BUILD_DIR/cmd/etl_worker app-traceroute.yaml
     && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-oti
@@ -432,6 +451,8 @@ deploy:
 - provider: script
   script:
     $TRAVIS_BUILD_DIR/travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_oti
+    && $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-oti
+    SERVICE_ACCOUNT_mlab_oti $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
     && gcloud app deploy --project=mlab-oti $TRAVIS_BUILD_DIR/appengine/queue.yaml
     && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-oti
     SERVICE_ACCOUNT_mlab_oti $TRAVIS_BUILD_DIR/cmd/etl_worker app-batch.yaml
@@ -448,6 +469,8 @@ deploy:
 - provider: script
   script:
     $TRAVIS_BUILD_DIR/travis/activate_service_account.sh SERVICE_ACCOUNT_mlab_oti
+    && $TRAVIS_BUILD_DIR/travis/run_with_application_credentials.sh mlab-oti
+    SERVICE_ACCOUNT_mlab_oti $TRAVIS_BUILD_DIR/cmd/update-schema update-schema
     && gcloud app deploy --project=mlab-oti $TRAVIS_BUILD_DIR/appengine/queue.yaml
     && $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-oti
     SERVICE_ACCOUNT_mlab_oti $TRAVIS_BUILD_DIR/cmd/etl_worker app-ndt.yaml
diff --git a/cmd/update-schema/update.go b/cmd/update-schema/update.go
@@ -119,12 +119,13 @@ func main() {
 		if err := CreateOrUpdatePT(project, "batch", "traceroute"); err != nil {
 			errCount++
 		}
+		/* Temporaily comment out due to NDT7 unstable schemas.
 		if err := CreateOrUpdateNDTResultRow(project, "base_tables", "ndt5"); err != nil {
 			errCount++
 		}
 		if err := CreateOrUpdateNDTResultRow(project, "batch", "ndt5"); err != nil {
 			errCount++
-		}
+		}*/
 
 	case "tcpinfo":
 		if err := CreateOrUpdateTCPInfo(project, "base_tables", "tcpinfo"); err != nil {
diff --git a/etl-schema b/etl-schema
@@ -1 +1 @@
-Subproject commit 7920a4c020a281daadf6d318e63670849dc21a79
+Subproject commit b9c30570dfd12d38e5f229a10d4891d0d4d6449e
diff --git a/travis b/travis
@@ -1 +1 @@
-Subproject commit 97156c01fb9f71d01726170a95c60b69ff183a82
+Subproject commit 4599d5d7875c77ce8a5e9c4582e1e20dbd269d93

Original file line number	Diff line number	Diff line change
`@@ -119,12 +119,13 @@ func main() {`
`119`	`119`	`if err := CreateOrUpdatePT(project, "batch", "traceroute"); err != nil {`
`120`	`120`	`errCount++`
`121`	`121`	`}`
	`122`	`+ /* Temporaily comment out due to NDT7 unstable schemas.`
`122`	`123`	`if err := CreateOrUpdateNDTResultRow(project, "base_tables", "ndt5"); err != nil {`
`123`	`124`	`errCount++`
`124`	`125`	`}`
`125`	`126`	`if err := CreateOrUpdateNDTResultRow(project, "batch", "ndt5"); err != nil {`
`126`	`127`	`errCount++`
`127`		`- }`
	`128`	`+ }*/`
`128`	`129`
`129`	`130`	`case "tcpinfo":`
`130`	`131`	`if err := CreateOrUpdateTCPInfo(project, "base_tables", "tcpinfo"); err != nil {`