Skip to content

Commit c1a37a1

Browse files
jingy-lijingy-li
authored
[test] Fix Argument List Too Long Issue in E2E Test DaVinciUserApp by Using Config File (linkedin#1980)
1 parent afa8a74 commit c1a37a1

3 files changed

Lines changed: 148 additions & 88 deletions

File tree

‎internal/venice-test-common/src/integrationTest/java/com/linkedin/davinci/DaVinciUserApp.java‎

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,11 @@
3737
import com.linkedin.venice.integration.utils.DaVinciTestContext;
3838
import com.linkedin.venice.utils.SslUtils;
3939
import io.tehuti.metrics.MetricsRepository;
40+
import java.io.FileInputStream;
4041
import java.util.HashMap;
4142
import java.util.Map;
4243
import java.util.Optional;
43-
import java.util.concurrent.ExecutionException;
44+
import java.util.Properties;
4445
import java.util.concurrent.TimeUnit;
4546
import org.apache.logging.log4j.LogManager;
4647
import org.apache.logging.log4j.Logger;
@@ -53,19 +54,31 @@
5354
public class DaVinciUserApp {
5455
private static final Logger LOGGER = LogManager.getLogger(DaVinciUserApp.class);
5556

56-
public static void main(String[] args) throws InterruptedException, ExecutionException {
57-
String zkHosts = args[0];
58-
String baseDataPath = args[1];
59-
String storeName = args[2];
60-
int sleepSeconds = Integer.parseInt(args[3]);
61-
int heartbeatTimeoutSeconds = Integer.parseInt(args[4]);
62-
boolean ingestionIsolation = Boolean.parseBoolean(args[5]);
63-
int blobTransferServerPort = Integer.parseInt(args[6]);
64-
int blobTransferClientPort = Integer.parseInt(args[7]);
65-
String storageClass = args[8]; // DISK or MEMORY_BACKED_BY_DISK
66-
boolean recordTransformerEnabled = Boolean.parseBoolean(args[9]);
67-
boolean blobTransferDaVinciSSLEnabled = Boolean.parseBoolean(args[10]);
68-
boolean batchPushReportEnabled = Boolean.parseBoolean(args[11]);
57+
public static void main(String[] args) throws Exception {
58+
if (args.length != 1) {
59+
throw new IllegalArgumentException("Expected config file path");
60+
}
61+
62+
// Load properties from file
63+
Properties props = new Properties();
64+
try (FileInputStream fis = new FileInputStream(args[0])) {
65+
props.load(fis);
66+
}
67+
68+
// Read properties
69+
String zkHosts = props.getProperty("zk.hosts");
70+
String baseDataPath = props.getProperty("base.data.path");
71+
String storeName = props.getProperty("store.name");
72+
int sleepSeconds = Integer.parseInt(props.getProperty("sleep.seconds"));
73+
int heartbeatTimeoutSeconds = Integer.parseInt(props.getProperty("heartbeat.timeout.seconds"));
74+
boolean ingestionIsolation = Boolean.parseBoolean(props.getProperty("ingestion.isolation"));
75+
int blobTransferServerPort = Integer.parseInt(props.getProperty("blob.transfer.server.port"));
76+
int blobTransferClientPort = Integer.parseInt(props.getProperty("blob.transfer.client.port"));
77+
String storageClass = props.getProperty("storage.class");
78+
boolean recordTransformerEnabled = Boolean.parseBoolean(props.getProperty("record.transformer.enabled"));
79+
boolean blobTransferDaVinciManagerEnabled =
80+
Boolean.parseBoolean(props.getProperty("blob.transfer.manager.enabled"));
81+
boolean batchPushReportEnabled = Boolean.parseBoolean(props.getProperty("batch.push.report.enabled"));
6982

7083
D2Client d2Client = new D2ClientBuilder().setZkHosts(zkHosts)
7184
.setZkSessionTimeout(3, TimeUnit.SECONDS)
@@ -77,12 +90,13 @@ public static void main(String[] args) throws InterruptedException, ExecutionExc
7790
extraBackendConfig.put(SERVER_INGESTION_MODE, ingestionIsolation ? ISOLATED : BUILT_IN);
7891
extraBackendConfig.put(SERVER_INGESTION_ISOLATION_CONNECTION_TIMEOUT_SECONDS, heartbeatTimeoutSeconds);
7992
extraBackendConfig.put(DATA_BASE_PATH, baseDataPath);
80-
extraBackendConfig.put(DAVINCI_P2P_BLOB_TRANSFER_SERVER_PORT, blobTransferServerPort);
81-
extraBackendConfig.put(DAVINCI_P2P_BLOB_TRANSFER_CLIENT_PORT, blobTransferClientPort);
8293
extraBackendConfig.put(PUSH_STATUS_STORE_ENABLED, true);
83-
extraBackendConfig.put(BLOB_TRANSFER_MANAGER_ENABLED, true);
8494

85-
if (blobTransferDaVinciSSLEnabled) {
95+
if (blobTransferDaVinciManagerEnabled) {
96+
extraBackendConfig.put(BLOB_TRANSFER_MANAGER_ENABLED, true);
97+
extraBackendConfig.put(DAVINCI_P2P_BLOB_TRANSFER_SERVER_PORT, blobTransferServerPort);
98+
extraBackendConfig.put(DAVINCI_P2P_BLOB_TRANSFER_CLIENT_PORT, blobTransferClientPort);
99+
86100
extraBackendConfig.put(BLOB_TRANSFER_SSL_ENABLED, true);
87101
extraBackendConfig.put(BLOB_TRANSFER_ACL_ENABLED, true);
88102

‎internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
import com.linkedin.venice.utils.VeniceProperties;
7777
import io.tehuti.metrics.MetricsRepository;
7878
import java.io.File;
79+
import java.io.FileWriter;
7980
import java.io.IOException;
8081
import java.nio.file.Files;
8182
import java.nio.file.Paths;
@@ -663,20 +664,28 @@ public void testBlobTransferRecordTransformer() throws Exception {
663664
LOGGER.info("zkHosts is {}", zkHosts);
664665

665666
// Start the first DaVinci Client using DaVinciUserApp for regular ingestion
666-
ForkedJavaProcess.exec(
667-
DaVinciUserApp.class,
668-
zkHosts,
669-
dvcPath1,
670-
storeName,
671-
"100",
672-
"10",
673-
"false",
674-
Integer.toString(port1),
675-
Integer.toString(port2),
676-
StorageClass.DISK.toString(),
677-
"true",
678-
"true",
679-
"false");
667+
File configDir = Utils.getTempDataDirectory();
668+
File configFile = new File(configDir, "dvc-config.properties");
669+
Properties props = new Properties();
670+
props.setProperty("zk.hosts", zkHosts);
671+
props.setProperty("base.data.path", dvcPath1);
672+
props.setProperty("store.name", storeName);
673+
props.setProperty("sleep.seconds", "100");
674+
props.setProperty("heartbeat.timeout.seconds", "10");
675+
props.setProperty("ingestion.isolation", "false");
676+
props.setProperty("blob.transfer.server.port", Integer.toString(port1));
677+
props.setProperty("blob.transfer.client.port", Integer.toString(port2));
678+
props.setProperty("storage.class", StorageClass.DISK.toString());
679+
props.setProperty("record.transformer.enabled", "true");
680+
props.setProperty("blob.transfer.manager.enabled", "true");
681+
props.setProperty("batch.push.report.enabled", "false");
682+
683+
// Write properties to file
684+
try (FileWriter writer = new FileWriter(configFile)) {
685+
props.store(writer, null);
686+
}
687+
688+
ForkedJavaProcess.exec(DaVinciUserApp.class, configFile.getAbsolutePath());
680689

681690
// Wait for the first DaVinci Client to complete ingestion
682691
Thread.sleep(60000);

‎internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java‎

Lines changed: 93 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@
129129
import io.tehuti.Metric;
130130
import io.tehuti.metrics.MetricsRepository;
131131
import java.io.File;
132+
import java.io.FileWriter;
132133
import java.io.IOException;
133134
import java.nio.ByteBuffer;
134135
import java.nio.file.Files;
@@ -1215,20 +1216,32 @@ public void testCrashedDaVinciWithIngestionIsolation() throws Exception {
12151216
while (port1 == port2) {
12161217
port2 = TestUtils.getFreePort();
12171218
}
1218-
ForkedJavaProcess forkedDaVinciUserApp = ForkedJavaProcess.exec(
1219-
DaVinciUserApp.class,
1220-
zkHosts,
1221-
baseDataPath,
1222-
storeName,
1223-
"100",
1224-
"10",
1225-
"true",
1226-
Integer.toString(port1),
1227-
Integer.toString(port2),
1228-
StorageClass.DISK.toString(),
1229-
"false",
1230-
"false",
1231-
"false");
1219+
1220+
// Start the first DaVinci Client using DaVinciUserApp for regular ingestion
1221+
File configDir = Utils.getTempDataDirectory();
1222+
File configFile = new File(configDir, "dvc-config.properties");
1223+
1224+
Properties props = new Properties();
1225+
props.setProperty("zk.hosts", zkHosts);
1226+
props.setProperty("base.data.path", baseDataPath);
1227+
props.setProperty("store.name", storeName);
1228+
props.setProperty("sleep.seconds", "100");
1229+
props.setProperty("heartbeat.timeout.seconds", "10");
1230+
props.setProperty("ingestion.isolation", "true");
1231+
props.setProperty("blob.transfer.server.port", Integer.toString(port1));
1232+
props.setProperty("blob.transfer.client.port", Integer.toString(port2));
1233+
props.setProperty("storage.class", StorageClass.DISK.toString());
1234+
props.setProperty("record.transformer.enabled", "false");
1235+
props.setProperty("blob.transfer.manager.enabled", "false");
1236+
props.setProperty("batch.push.report.enabled", "false");
1237+
1238+
// Write properties to file
1239+
try (FileWriter writer = new FileWriter(configFile)) {
1240+
props.store(writer, null);
1241+
}
1242+
1243+
ForkedJavaProcess forkedDaVinciUserApp = ForkedJavaProcess.exec(DaVinciUserApp.class, configFile.getAbsolutePath());
1244+
12321245
// Sleep long enough so the forked Da Vinci app process can finish ingestion.
12331246
Thread.sleep(60000);
12341247
IsolatedIngestionUtils.executeShellCommand("kill " + forkedDaVinciUserApp.pid());
@@ -1293,20 +1306,28 @@ public void testBlobP2PTransferAmongDVC(boolean batchPushReportEnable) throws Ex
12931306
setUpStore(storeName, paramsConsumer, properties -> {}, true);
12941307

12951308
// Start the first DaVinci Client using DaVinciUserApp for regular ingestion
1296-
ForkedJavaProcess.exec(
1297-
DaVinciUserApp.class,
1298-
zkHosts,
1299-
dvcPath1,
1300-
storeName,
1301-
"100",
1302-
"10",
1303-
"false",
1304-
Integer.toString(port1),
1305-
Integer.toString(port2),
1306-
StorageClass.DISK.toString(),
1307-
"false",
1308-
"true",
1309-
String.valueOf(batchPushReportEnable));
1309+
File configDir = Utils.getTempDataDirectory();
1310+
File configFile = new File(configDir, "dvc-config.properties");
1311+
Properties props = new Properties();
1312+
props.setProperty("zk.hosts", zkHosts);
1313+
props.setProperty("base.data.path", dvcPath1);
1314+
props.setProperty("store.name", storeName);
1315+
props.setProperty("sleep.seconds", "100");
1316+
props.setProperty("heartbeat.timeout.seconds", "10");
1317+
props.setProperty("ingestion.isolation", "false");
1318+
props.setProperty("blob.transfer.server.port", Integer.toString(port1));
1319+
props.setProperty("blob.transfer.client.port", Integer.toString(port2));
1320+
props.setProperty("storage.class", StorageClass.DISK.toString());
1321+
props.setProperty("record.transformer.enabled", "false");
1322+
props.setProperty("blob.transfer.manager.enabled", "true");
1323+
props.setProperty("batch.push.report.enabled", String.valueOf(batchPushReportEnable));
1324+
1325+
// Write properties to file
1326+
try (FileWriter writer = new FileWriter(configFile)) {
1327+
props.store(writer, null);
1328+
}
1329+
1330+
ForkedJavaProcess.exec(DaVinciUserApp.class, configFile.getAbsolutePath());
13101331

13111332
// Wait for the first DaVinci Client to complete ingestion
13121333
Thread.sleep(60000);
@@ -1422,20 +1443,28 @@ public void testBlobP2PTransferForNonLaggingDaVinciClient() throws Exception {
14221443
setUpStore(storeName, paramsConsumer, properties -> {}, true);
14231444

14241445
// Start the first DaVinci Client using DaVinciUserApp for regular ingestion
1425-
ForkedJavaProcess.exec(
1426-
DaVinciUserApp.class,
1427-
zkHosts,
1428-
dvcPath1,
1429-
storeName,
1430-
"100",
1431-
"10",
1432-
"false",
1433-
Integer.toString(port1),
1434-
Integer.toString(port2),
1435-
StorageClass.DISK.toString(),
1436-
"false",
1437-
"true",
1438-
"false");
1446+
File configDir = Utils.getTempDataDirectory();
1447+
File configFile = new File(configDir, "dvc-config.properties");
1448+
Properties props = new Properties();
1449+
props.setProperty("zk.hosts", zkHosts);
1450+
props.setProperty("base.data.path", dvcPath1);
1451+
props.setProperty("store.name", storeName);
1452+
props.setProperty("sleep.seconds", "100");
1453+
props.setProperty("heartbeat.timeout.seconds", "10");
1454+
props.setProperty("ingestion.isolation", "false");
1455+
props.setProperty("blob.transfer.server.port", Integer.toString(port1));
1456+
props.setProperty("blob.transfer.client.port", Integer.toString(port2));
1457+
props.setProperty("storage.class", StorageClass.DISK.toString());
1458+
props.setProperty("record.transformer.enabled", "false");
1459+
props.setProperty("blob.transfer.manager.enabled", "true");
1460+
props.setProperty("batch.push.report.enabled", "false");
1461+
1462+
// Write properties to file
1463+
try (FileWriter writer = new FileWriter(configFile)) {
1464+
props.store(writer, null);
1465+
}
1466+
1467+
ForkedJavaProcess.exec(DaVinciUserApp.class, configFile.getAbsolutePath());
14391468

14401469
// Wait for the first DaVinci Client to complete ingestion
14411470
Thread.sleep(60000);
@@ -1609,20 +1638,28 @@ public void testBlobP2PTransferAmongDVCWithServerShutdown(boolean isGracefulShut
16091638
setUpStore(storeName, paramsConsumer, properties -> {}, true);
16101639

16111640
// Start the first DaVinci Client using DaVinciUserApp
1612-
ForkedJavaProcess forkedDaVinciUserApp = ForkedJavaProcess.exec(
1613-
DaVinciUserApp.class,
1614-
zkHosts,
1615-
dvcPath1,
1616-
storeName,
1617-
"100",
1618-
"10",
1619-
"false",
1620-
Integer.toString(port1),
1621-
Integer.toString(port2),
1622-
StorageClass.DISK.toString(),
1623-
"false",
1624-
"true",
1625-
"false");
1641+
File configDir = Utils.getTempDataDirectory();
1642+
File configFile = new File(configDir, "dvc-config.properties");
1643+
Properties props = new Properties();
1644+
props.setProperty("zk.hosts", zkHosts);
1645+
props.setProperty("base.data.path", dvcPath1);
1646+
props.setProperty("store.name", storeName);
1647+
props.setProperty("sleep.seconds", "100");
1648+
props.setProperty("heartbeat.timeout.seconds", "10");
1649+
props.setProperty("ingestion.isolation", "false");
1650+
props.setProperty("blob.transfer.server.port", Integer.toString(port1));
1651+
props.setProperty("blob.transfer.client.port", Integer.toString(port2));
1652+
props.setProperty("storage.class", StorageClass.DISK.toString());
1653+
props.setProperty("record.transformer.enabled", "false");
1654+
props.setProperty("blob.transfer.manager.enabled", "true");
1655+
props.setProperty("batch.push.report.enabled", "false");
1656+
1657+
// Write properties to file
1658+
try (FileWriter writer = new FileWriter(configFile)) {
1659+
props.store(writer, null);
1660+
}
1661+
1662+
ForkedJavaProcess forkedDaVinciUserApp = ForkedJavaProcess.exec(DaVinciUserApp.class, configFile.getAbsolutePath());
16261663

16271664
// Wait for the first DaVinci Client to complete ingestion
16281665
Thread.sleep(60000);

0 commit comments

Comments
 (0)