traceroute tuning (#724)

* Update app-traceroute.yaml

* Update app-batch.yaml

* Update app-traceroute.yaml

* Update queue.yaml

* Update app-batch.yaml

* fix

* more

* Update globals.go

* add concurrency

* Update globals.go

* Update app-batch.yaml

* Update app-batch.yaml

Author: yachang

appengine/queue.yaml

@@ -468,7 +468,7 @@ queue:
   rate: 1/s
   bucket_size: 10
   # max concurrent is limited, because we will share the pipeline with other experiment types
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     # min and max are same, so that queue rate only diminishes as tasks are drained
@@ -480,7 +480,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -489,7 +489,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -498,7 +498,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -507,7 +507,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -516,7 +516,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -525,7 +525,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -534,7 +534,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -543,7 +543,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -552,7 +552,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -561,7 +561,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -570,7 +570,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -579,7 +579,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -588,7 +588,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -597,7 +597,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20
@@ -606,7 +606,7 @@ queue:
   target: etl-batch-parser
   rate: 1/s
   bucket_size: 10
-  max_concurrent_requests: 5
+  max_concurrent_requests: 20
   retry_parameters:
     task_age_limit: 12h
     min_backoff_seconds: 20

etl/globals.go

@@ -255,7 +255,7 @@ var (
 		NDT_OMIT_DELTAS: 50,
 		TCPINFO:         10,
 		SS:              500, // Average json size is 2.5K
-		PT:              5,
+		PT:              100,
 		SW:              100,
 		NDT5:            50,
 		NDT7:            50,