Add a GCS project flag (#20)

* Add GCP project flag

* Use PROJECT_ID variable

* Rename GCS

Author: cristinaleonr

cloudbuild.yaml

@@ -35,6 +35,8 @@ steps:
   - |-
     sed -i -e 's/{{COMMIT_SHA}}/'${COMMIT_SHA}'/' \
            -e 's/{{PROJECT_ID}}/'${PROJECT_ID}'/' \
+           -e 's/{{VIEW_PROJECT}}/'$_VIEW_PROJECT'/' \
+           -e 's/{{GCS_PROJECT}}/'$_GCS_PROJECT'/' \
            -e 's/{{MLAB_BUCKET}}/'$_MLAB_BUCKET'/' \
            -e 's/{{BUCKETS}}/'$_BUCKETS'/' \
            k8s/autoloader.yaml

cmd/autoloader/main.go

@@ -17,17 +17,19 @@ import (
 
 var (
 	listenAddr          string
-	project             string
+	bqProject           string
 	viewProject         string
+	gcsProject          string
 	mlabBucket          string
 	bucketNames         flagx.StringArray
 	mainCtx, mainCancel = context.WithCancel(context.Background())
 )
 
 func init() {
 	flag.StringVar(&listenAddr, "listenaddr", ":8080", "Address to listen on")
-	flag.StringVar(&project, "project", "mlab-sandbox", "BigQuery project environment variable")
+	flag.StringVar(&bqProject, "bq-project", "mlab-sandbox", "BigQuery project environment variable")
 	flag.StringVar(&viewProject, "view-project", "mlab-sandbox", "BigQuery project for views")
+	flag.StringVar(&gcsProject, "gcs-project", "mlab-sandbox", "GCS project")
 	flag.StringVar(&mlabBucket, "mlab-bucket", "", "Archive bucket name containing data from M-Lab's platform")
 	flag.Var(&bucketNames, "buckets", "Archive bucket names in Google Cloud Storage")
 }
@@ -43,14 +45,14 @@ func main() {
 	storage, err := storage.NewClient(mainCtx)
 	rtx.Must(err, "Failed to create storage client")
 	defer storage.Close()
-	gcs := gcs.NewClient(storage, bucketNames, mlabBucket, project)
+	gcs := gcs.NewClient(storage, bucketNames, mlabBucket, gcsProject)
 
-	bqMain, err := bigquery.NewClient(mainCtx, project)
+	bqMain, err := bigquery.NewClient(mainCtx, bqProject)
 	rtx.Must(err, "Failed to create BigQuery client")
 	defer bqMain.Close()
 
 	bqView := bqMain
-	if project != viewProject {
+	if bqProject != viewProject {
 		bqView, err = bigquery.NewClient(mainCtx, viewProject)
 		rtx.Must(err, "Failed to create BigQuery view client")
 		defer bqView.Close()

k8s/autoloader.yaml

@@ -31,7 +31,9 @@ spec:
         image: gcr.io/{{PROJECT_ID}}/autoloader:{{COMMIT_SHA}}
         args:
         - -listenaddr=:8080
-        - -project={{PROJECT_ID}}
+        - -bq-project={{PROJECT_ID}}
+        - -view-project={{VIEW_PROJECT}}
+        - -gcs-project={{GCS_PROJECT}}
         - -mlab-bucket={{MLAB_BUCKET}}
         - -buckets={{BUCKETS}}
         - -prometheusx.listen-address=:9990