Add separate config for etl-ndt-staging

Author: gfr10598

.travis.yml

@@ -95,7 +95,7 @@ deploy:
 # when *ANY* branch is tagged with ndt-staging-*'
 - provider: script
   script: $TRAVIS_BUILD_DIR/travis/deploy_app.sh mlab-staging
-    /tmp/mlab-staging.json $TRAVIS_BUILD_DIR/cmd/etl_worker app-ndt.yaml
+    /tmp/mlab-staging.json $TRAVIS_BUILD_DIR/cmd/etl_worker app-ndt-staging.yaml
   skip_cleanup: true
   on:
     repo: m-lab/etl

cmd/etl_worker/app-ndt-staging.yaml

@@ -0,0 +1,48 @@
+# To deploy (from cmd/etl_worker):
+# go build etl_worker.go && gcloud beta app deploy --project=mlab-staging app-ndt-staging.yaml
+
+runtime: custom
+env: flex
+service: etl-ndt-parser
+
+# Resource and scaling options. For more background, see:
+#   https://cloud.google.com/appengine/docs/flexible/go/configuring-your-app-with-app-yaml
+
+# TODO(dev): adjust CPU and memory based on actual requirements.
+resources:
+  cpu: 2
+  # Instances support between [(cpu * 0.9) - 0.4, (cpu * 6.5) - 0.4]
+  # Actual memory available is exposed via GAE_MEMORY_MB environment variable.
+  memory_gb: 6
+
+  # TODO - Adjust once we understand requirements.
+  disk_size_gb: 10
+
+automatic_scaling:
+  # This is intended for batch jobs.  A single instance is required, and we allow
+  # it to scale up to 50 instances to get work done quickly.  However, note that
+  # more than 1.5 tasks/sec may result in bigquery stream quota problems, so this
+  # may require changes.
+  min_num_instances: 1
+  max_num_instances: 50
+  cool_down_period_sec: 300
+  # We don't care much about latency, so a high utilization is desireable.
+  cpu_utilization:
+    target_utilization: 0.85
+
+# Note: add a public port for GCE auto discovery by prometheus.
+# TODO(dev): are any values redundant or irrelevant?
+network:
+  instance_tag: etl-parser
+  name: default
+  # Forward port 9090 on the GCE instance address to the same port in the
+  # container address. Only forward TCP traffic.
+  # Note: the default AppEngine container port 8080 cannot be forwarded.
+  forwarded_ports:
+    - 9090/tcp
+
+env_variables:
+  MAX_WORKERS: 25
+  BIGQUERY_PROJECT: 'mlab-staging'
+  BIGQUERY_DATASET: 'staging'
+  # TODO add custom service-account, instead of using default credentials.