Skip to content
This repository was archived by the owner on Dec 4, 2024. It is now read-only.

Commit 58323dd

Browse files
mpereirampereira
authored
[DCOS-59736] Add MWT #21 parameter files. (#552)
1 parent b9a3e4a commit 58323dd

File tree

2 files changed

+340
-0
lines changed

2 files changed

+340
-0
lines changed

‎scale-tests/configs/2019-10-09-mwt21dr.env

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
# Depends on:
2+
# - TEST_NAME
3+
# - TEST_S3_BUCKET
4+
# - TEST_S3_FOLDER
5+
6+
# Workload configuration #######################################################
7+
#
8+
# Total CPU quota: 88
9+
# Total MEM quota: 200000
10+
# Total GPU quota: 40
11+
12+
CLUSTER_URL="https://mwt21dr.scaletesting.mesosphe.re"
13+
SECURITY="strict"
14+
15+
DCOS_CLI_URL="https://downloads.dcos.io/cli/releases/binaries/dcos/linux/x86-64/latest/dcos"
16+
17+
# Test configuration ###########################################################
18+
19+
SHOULD_INSTALL_INFRASTRUCTURE=true
20+
SHOULD_INSTALL_NON_GPU_DISPATCHERS=true
21+
SHOULD_INSTALL_GPU_DISPATCHERS=false
22+
SHOULD_RUN_FINITE_STREAMING_JOBS=true
23+
SHOULD_RUN_INFINITE_STREAMING_JOBS=true
24+
SHOULD_RUN_BATCH_JOBS=true
25+
SHOULD_RUN_GPU_BATCH_JOBS=false
26+
SHOULD_UNINSTALL_INFRASTRUCTURE_AT_THE_END=false
27+
28+
# Infrastructure configuration #################################################
29+
30+
GROUP_NAME="${TEST_NAME}"
31+
32+
SERVICE_NAMES_PREFIX="${TEST_NAME}/"
33+
INFRASTRUCTURE_OUTPUT_FILE="infrastructure.json"
34+
35+
KAFKA_CLUSTER_COUNT=1
36+
CASSANDRA_CLUSTER_COUNT=1
37+
38+
ZOOKEEPER_CPUS=10
39+
ZOOKEEPER_MEM=20000
40+
ZOOKEEPER_CONFIG='scale-tests/configs/kafka-zookeeper-options.json'
41+
# Note: empty package repo values will default to latest Universe packages.
42+
ZOOKEEPER_PACKAGE_REPO="https://universe-converter.mesosphere.com/transform?url=https://infinity-artifacts.s3.amazonaws.com/permanent/confluent-zookeeper/assets/2.6.1-5.1.2e/stub-universe-confluent-zookeeper.json"
43+
# 2.6.0-5.1.2e from the Universe didn't have role-enforcement support. Used
44+
# 2.6.1-5.1.2e permanent stub above.
45+
46+
KAFKA_CPUS=10
47+
KAFKA_MEM=20000
48+
KAFKA_CONFIG='scale-tests/configs/kafka-options.json'
49+
# Note: empty package repo values will default to latest Universe packages.
50+
KAFKA_PACKAGE_REPO="https://universe-converter.mesosphere.com/transform?url=https://infinity-artifacts.s3.amazonaws.com/permanent/confluent-kafka/assets/2.7.1-5.3.0/stub-universe-confluent-kafka.json"
51+
# 2.7.0-5.3.0 from the Universe didn't have role-enforcement support. Used
52+
# 2.7.1-5.3.0 permanent stub above.
53+
54+
CASSANDRA_CPUS=10
55+
CASSANDRA_MEM=20000
56+
CASSANDRA_CONFIG='scale-tests/configs/cassandra-options.json'
57+
# Note: empty package repo values will default to latest Universe packages.
58+
CASSANDRA_PACKAGE_REPO=
59+
# 2.7.0-3.11.4 from the Universe.
60+
61+
# DSEngine configuration #######################################################
62+
63+
DSENGINE_CPUS=10
64+
DSENGINE_MEM=20000
65+
DSENGINE_GPUS=40
66+
DSENGINE_PACKAGE_REPO=
67+
68+
# Spark configuration ##########################################################
69+
70+
SPARK_CONFIG='scale-tests/configs/spark-options.json'
71+
72+
# Note: empty package repo values will default to latest Universe packages.
73+
# 2.9.0-2.4.3 from the Universe.
74+
SPARK_PACKAGE_REPO=
75+
76+
# Note: leaving the Spark executor Docker image empty so that executors inherit
77+
# the image used for dispatchers.
78+
SPARK_EXECUTOR_DOCKER_IMAGE=
79+
80+
# Non-GPU Spark dispatchers configuration ######################################
81+
82+
# Not currently used.
83+
BATCH_MAX_NON_GPU_JOBS=30
84+
85+
SPARK_NON_GPU_DISPATCHERS=3
86+
SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE="non-gpu-dispatchers.out"
87+
# Note: this name is built internally by the deploy-dispatchers.py script.
88+
SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json"
89+
# Note: driver resources used per dispatcher (1 dispatcher will be able to run
90+
# 8 drivers since each driver requires 1 CPU).
91+
SPARK_NON_GPU_QUOTA_DRIVERS_CPUS=8
92+
SPARK_NON_GPU_QUOTA_DRIVERS_MEM=20000
93+
# Note: executor resources used per job (1 driver will run 1 job).
94+
SPARK_NON_GPU_QUOTA_EXECUTORS_CPUS=8
95+
SPARK_NON_GPU_QUOTA_EXECUTORS_MEM=20000
96+
97+
# GPU Spark dispatchers configuration ##########################################
98+
99+
# Not currently used.
100+
BATCH_MAX_GPU_JOBS=2
101+
102+
SPARK_GPU_DISPATCHERS=0
103+
SPARK_GPU_DISPATCHERS_OUTPUT_FILE="gpu-dispatchers.out"
104+
SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json" # NOTE: this name is built internally by the deploy-dispatchers.py script.
105+
SPARK_GPU_QUOTA_DRIVERS_CPUS=
106+
SPARK_GPU_QUOTA_DRIVERS_MEM=
107+
SPARK_GPU_QUOTA_DRIVERS_GPUS=
108+
SPARK_GPU_QUOTA_EXECUTORS_CPUS=
109+
SPARK_GPU_QUOTA_EXECUTORS_MEM=
110+
SPARK_GPU_QUOTA_EXECUTORS_GPUS=
111+
112+
# Common streaming jobs configuration ##########################################
113+
114+
TEST_ASSEMBLY_JAR_URL='http://infinity-artifacts.s3.amazonaws.com/scale-tests/dcos-spark-scala-tests-assembly-2.4.0-20190325.jar'
115+
DISPATCHERS_JSON_OUTPUT_FILE="all-dispatchers.json"
116+
117+
# Finite streaming jobs configuration ##########################################
118+
119+
STREAMING_FINITE_SUBMISSIONS_OUTPUT_FILE="finite-submissions.out"
120+
STREAMING_FINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 3 dispatchers -> 3 producers.
121+
STREAMING_FINITE_CONSUMERS_PER_PRODUCER=1 # 3 producers -> 3 consumers.
122+
# 3 producers + 3 consumers = 6 total finite streaming jobs
123+
STREAMING_FINITE_PRODUCER_NUMBER_OF_WORDS=7692
124+
STREAMING_FINITE_PRODUCER_WORDS_PER_SECOND=1
125+
# 7692 words / 1 word per second -> ~2h runtime.
126+
STREAMING_FINITE_PRODUCER_SPARK_CORES_MAX=2
127+
STREAMING_FINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
128+
STREAMING_FINITE_CONSUMER_BATCH_SIZE_SECONDS=10
129+
STREAMING_FINITE_CONSUMER_SPARK_CORES_MAX=1
130+
STREAMING_FINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
131+
132+
# Infinite streaming jobs configuration ########################################
133+
134+
STREAMING_INFINITE_SUBMISSIONS_OUTPUT_FILE="infinite-submissions.out"
135+
STREAMING_INFINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 3 dispatchers -> 3 producers.
136+
STREAMING_INFINITE_CONSUMERS_PER_PRODUCER=1 # 3 producers -> 3 consumers.
137+
# 3 producers + 3 consumers = 6 total infinite streaming jobs
138+
STREAMING_INFINITE_PRODUCER_NUMBER_OF_WORDS=0
139+
STREAMING_INFINITE_PRODUCER_WORDS_PER_SECOND=1
140+
STREAMING_INFINITE_PRODUCER_SPARK_CORES_MAX=2
141+
STREAMING_INFINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
142+
STREAMING_INFINITE_CONSUMER_BATCH_SIZE_SECONDS=10
143+
STREAMING_INFINITE_CONSUMER_SPARK_CORES_MAX=1
144+
STREAMING_INFINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
145+
146+
# Batch jobs configuration #####################################################
147+
148+
SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
149+
150+
BATCH_APP_ID="/${SERVICE_NAMES_PREFIX}batch-workload"
151+
BATCH_SCRIPT_CPUS=6
152+
BATCH_SCRIPT_MEM=12288
153+
BATCH_SUBMITS_PER_MIN=13
154+
# TODO: update to master for the next MWT.
155+
BATCH_SPARK_BUILD_BRANCH="dcos-58437-deploy-workloads-under-role-enforced-group"
156+
157+
# Batch GPU jobs configuration #################################################
158+
159+
SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
160+
161+
GPU_APP_ID="/${SERVICE_NAMES_PREFIX}gpu-batch-workload"
162+
GPU_SCRIPT_CPUS=2
163+
GPU_SCRIPT_MEM=4096
164+
GPU_DOCKER_IMAGE='samvantran/spark-dcos-gpu:metrics'
165+
GPU_SUBMITS_PER_MIN=5
166+
GPU_MAX_DISPATCHERS=${SPARK_GPU_DISPATCHERS}
167+
GPU_SPARK_CORES_MAX=4
168+
GPU_SPARK_MESOS_EXECUTOR_GPUS=4
169+
GPU_SPARK_MESOS_MAX_GPUS=4
170+
GPU_SPARK_BUILD_BRANCH=master

‎scale-tests/configs/2019-10-10-mwt21.env

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
# Depends on:
2+
# - TEST_NAME
3+
# - TEST_S3_BUCKET
4+
# - TEST_S3_FOLDER
5+
6+
# Workload configuration #######################################################
7+
#
8+
# Total CPU quota: 2290
9+
# Total MEM quota: 4580000
10+
# Total GPU quota: 40
11+
12+
CLUSTER_URL="https://mwt21.scaletesting.mesosphe.re"
13+
SECURITY="strict"
14+
15+
DCOS_CLI_URL="https://downloads.dcos.io/cli/releases/binaries/dcos/linux/x86-64/latest/dcos"
16+
17+
# Test configuration ###########################################################
18+
19+
SHOULD_INSTALL_INFRASTRUCTURE=true
20+
SHOULD_INSTALL_NON_GPU_DISPATCHERS=true
21+
SHOULD_INSTALL_GPU_DISPATCHERS=false
22+
SHOULD_RUN_FINITE_STREAMING_JOBS=true
23+
SHOULD_RUN_INFINITE_STREAMING_JOBS=true
24+
SHOULD_RUN_BATCH_JOBS=true
25+
SHOULD_RUN_GPU_BATCH_JOBS=false
26+
SHOULD_UNINSTALL_INFRASTRUCTURE_AT_THE_END=false
27+
28+
# Infrastructure configuration #################################################
29+
30+
GROUP_NAME="${TEST_NAME}"
31+
32+
SERVICE_NAMES_PREFIX="${TEST_NAME}/"
33+
INFRASTRUCTURE_OUTPUT_FILE="infrastructure.json"
34+
35+
KAFKA_CLUSTER_COUNT=1
36+
CASSANDRA_CLUSTER_COUNT=1
37+
38+
ZOOKEEPER_CPUS=10
39+
ZOOKEEPER_MEM=20000
40+
ZOOKEEPER_CONFIG='scale-tests/configs/kafka-zookeeper-options.json'
41+
# Note: empty package repo values will default to latest Universe packages.
42+
ZOOKEEPER_PACKAGE_REPO="https://universe-converter.mesosphere.com/transform?url=https://infinity-artifacts.s3.amazonaws.com/permanent/confluent-zookeeper/assets/2.6.1-5.1.2e/stub-universe-confluent-zookeeper.json"
43+
# 2.6.0-5.1.2e from the Universe didn't have role-enforcement support. Used
44+
# 2.6.1-5.1.2e permanent stub above.
45+
46+
KAFKA_CPUS=10
47+
KAFKA_MEM=20000
48+
KAFKA_CONFIG='scale-tests/configs/kafka-options.json'
49+
# Note: empty package repo values will default to latest Universe packages.
50+
KAFKA_PACKAGE_REPO="https://universe-converter.mesosphere.com/transform?url=https://infinity-artifacts.s3.amazonaws.com/permanent/confluent-kafka/assets/2.7.1-5.3.0/stub-universe-confluent-kafka.json"
51+
# 2.7.0-5.3.0 from the Universe didn't have role-enforcement support. Used
52+
# 2.7.1-5.3.0 permanent stub above.
53+
54+
CASSANDRA_CPUS=10
55+
CASSANDRA_MEM=20000
56+
CASSANDRA_CONFIG='scale-tests/configs/cassandra-options.json'
57+
# Note: empty package repo values will default to latest Universe packages.
58+
CASSANDRA_PACKAGE_REPO=
59+
# 2.7.0-3.11.4 from the Universe.
60+
61+
# DSEngine configuration #######################################################
62+
63+
DSENGINE_CPUS=10
64+
DSENGINE_MEM=20000
65+
DSENGINE_GPUS=40
66+
DSENGINE_PACKAGE_REPO=
67+
68+
# Spark configuration ##########################################################
69+
70+
SPARK_CONFIG='scale-tests/configs/spark-options.json'
71+
72+
# Note: empty package repo values will default to latest Universe packages.
73+
# 2.9.0-2.4.3 from the Universe.
74+
SPARK_PACKAGE_REPO=
75+
76+
# Note: leaving the Spark executor Docker image empty so that executors inherit
77+
# the image used for dispatchers.
78+
SPARK_EXECUTOR_DOCKER_IMAGE=
79+
80+
# Non-GPU Spark dispatchers configuration ######################################
81+
82+
# Not currently used.
83+
BATCH_MAX_NON_GPU_JOBS=1000
84+
85+
SPARK_NON_GPU_DISPATCHERS=50
86+
SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE="non-gpu-dispatchers.out"
87+
# Note: this name is built internally by the deploy-dispatchers.py script.
88+
SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_NON_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json"
89+
# Note: driver resources used per dispatcher (1 dispatcher will be able to run
90+
# 20 drivers since each driver requires 1 CPU).
91+
SPARK_NON_GPU_QUOTA_DRIVERS_CPUS=20
92+
SPARK_NON_GPU_QUOTA_DRIVERS_MEM=50000
93+
# Note: executor resources used per job (1 driver will run 1 job).
94+
SPARK_NON_GPU_QUOTA_EXECUTORS_CPUS=25
95+
SPARK_NON_GPU_QUOTA_EXECUTORS_MEM=40000
96+
97+
# GPU Spark dispatchers configuration ##########################################
98+
99+
# Not currently used.
100+
BATCH_MAX_GPU_JOBS=10
101+
102+
SPARK_GPU_DISPATCHERS=0
103+
SPARK_GPU_DISPATCHERS_OUTPUT_FILE="gpu-dispatchers.out"
104+
SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE="${SPARK_GPU_DISPATCHERS_OUTPUT_FILE}-dispatchers.json" # NOTE: this name is built internally by the deploy-dispatchers.py script.
105+
SPARK_GPU_QUOTA_DRIVERS_CPUS=
106+
SPARK_GPU_QUOTA_DRIVERS_MEM=
107+
SPARK_GPU_QUOTA_DRIVERS_GPUS=
108+
SPARK_GPU_QUOTA_EXECUTORS_CPUS=
109+
SPARK_GPU_QUOTA_EXECUTORS_MEM=
110+
SPARK_GPU_QUOTA_EXECUTORS_GPUS=
111+
112+
# Common streaming jobs configuration ##########################################
113+
114+
TEST_ASSEMBLY_JAR_URL='http://infinity-artifacts.s3.amazonaws.com/scale-tests/dcos-spark-scala-tests-assembly-2.4.0-20190325.jar'
115+
DISPATCHERS_JSON_OUTPUT_FILE="all-dispatchers.json"
116+
117+
# Finite streaming jobs configuration ##########################################
118+
119+
STREAMING_FINITE_SUBMISSIONS_OUTPUT_FILE="finite-submissions.out"
120+
STREAMING_FINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 50 dispatchers -> 50 producers.
121+
STREAMING_FINITE_CONSUMERS_PER_PRODUCER=1 # 50 producers -> 50 consumers.
122+
# 50 producers + 50 consumers = 100 total finite streaming jobs
123+
STREAMING_FINITE_PRODUCER_NUMBER_OF_WORDS=7692
124+
STREAMING_FINITE_PRODUCER_WORDS_PER_SECOND=1
125+
# 7692 words / 1 word per second -> ~2h runtime.
126+
STREAMING_FINITE_PRODUCER_SPARK_CORES_MAX=2
127+
STREAMING_FINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
128+
STREAMING_FINITE_CONSUMER_BATCH_SIZE_SECONDS=10
129+
STREAMING_FINITE_CONSUMER_SPARK_CORES_MAX=1
130+
STREAMING_FINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
131+
132+
# Infinite streaming jobs configuration ########################################
133+
134+
STREAMING_INFINITE_SUBMISSIONS_OUTPUT_FILE="infinite-submissions.out"
135+
STREAMING_INFINITE_PRODUCERS_PER_KAFKA="${SPARK_NON_GPU_DISPATCHERS}" # 1 Kafka and 50 dispatchers -> 50 producers.
136+
STREAMING_INFINITE_CONSUMERS_PER_PRODUCER=1 # 50 producers -> 50 consumers.
137+
# 50 producers + 50 consumers = 100 total infinite streaming jobs
138+
STREAMING_INFINITE_PRODUCER_NUMBER_OF_WORDS=0
139+
STREAMING_INFINITE_PRODUCER_WORDS_PER_SECOND=1
140+
STREAMING_INFINITE_PRODUCER_SPARK_CORES_MAX=2
141+
STREAMING_INFINITE_PRODUCER_SPARK_EXECUTOR_CORES=2
142+
STREAMING_INFINITE_CONSUMER_BATCH_SIZE_SECONDS=10
143+
STREAMING_INFINITE_CONSUMER_SPARK_CORES_MAX=1
144+
STREAMING_INFINITE_CONSUMER_SPARK_EXECUTOR_CORES=1
145+
146+
# Batch jobs configuration #####################################################
147+
148+
SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_NON_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
149+
150+
BATCH_APP_ID="/${SERVICE_NAMES_PREFIX}batch-workload"
151+
BATCH_SCRIPT_CPUS=6
152+
BATCH_SCRIPT_MEM=12288
153+
BATCH_SUBMITS_PER_MIN=13
154+
# TODO: update to master for the next MWT.
155+
BATCH_SPARK_BUILD_BRANCH="dcos-58437-deploy-workloads-under-role-enforced-group"
156+
157+
# Batch GPU jobs configuration #################################################
158+
159+
SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE_URL="https://${TEST_S3_BUCKET}.s3.amazonaws.com/${TEST_S3_FOLDER}/${SPARK_GPU_DISPATCHERS_JSON_OUTPUT_FILE}"
160+
161+
GPU_APP_ID="/${SERVICE_NAMES_PREFIX}gpu-batch-workload"
162+
GPU_SCRIPT_CPUS=2
163+
GPU_SCRIPT_MEM=4096
164+
GPU_DOCKER_IMAGE='samvantran/spark-dcos-gpu:metrics'
165+
GPU_SUBMITS_PER_MIN=5
166+
GPU_MAX_DISPATCHERS=${SPARK_GPU_DISPATCHERS}
167+
GPU_SPARK_CORES_MAX=4
168+
GPU_SPARK_MESOS_EXECUTOR_GPUS=4
169+
GPU_SPARK_MESOS_MAX_GPUS=4
170+
GPU_SPARK_BUILD_BRANCH=master

0 commit comments

Comments
 (0)