Skip to content

Commit a62898c

Browse files
azinneeraazinneera
authored
Merge pull request #43009 from azinneera/2201.9.x
Improve compile-time memory consumption for large projects
2 parents 2a016c1 + 1708ad3 commit a62898c

23 files changed

+234
-44
lines changed

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/BuildCommand.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,10 @@ public BuildCommand() {
198198
"generation")
199199
private String graalVMBuildOptions;
200200

201+
@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
202+
description = "experimental memory optimization for large projects")
203+
private Boolean optimizeDependencyCompilation;
204+
201205
public void execute() {
202206
long start = 0;
203207
if (this.helpFlag) {
@@ -316,7 +320,8 @@ private BuildOptions constructBuildOptions() {
316320
.setNativeImage(nativeImage)
317321
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
318322
.setGraalVMBuildOptions(graalVMBuildOptions)
319-
.setShowDependencyDiagnostics(showDependencyDiagnostics);
323+
.setShowDependencyDiagnostics(showDependencyDiagnostics)
324+
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);
320325

321326
if (targetDir != null) {
322327
buildOptionsBuilder.targetDir(targetDir.toString());

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/CommandUtil.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.gson.Gson;
2222
import com.google.gson.JsonArray;
2323
import com.google.gson.JsonObject;
24+
import io.ballerina.projects.BuildOptions;
2425
import io.ballerina.projects.JBallerinaBackend;
2526
import io.ballerina.projects.JvmTarget;
2627
import io.ballerina.projects.Package;
@@ -1156,9 +1157,11 @@ static String getLatestVersion(List<String> versions) {
11561157
* @param orgName org name of the dependent package
11571158
* @param packageName name of the dependent package
11581159
* @param version version of the dependent package
1160+
* @param buildOptions build options {sticky, offline}
11591161
* @return true if the dependent package compilation has errors
11601162
*/
1161-
static boolean pullDependencyPackages(String orgName, String packageName, String version) {
1163+
static boolean pullDependencyPackages(String orgName, String packageName, String version,
1164+
BuildOptions buildOptions) {
11621165
Path ballerinaUserHomeDirPath = ProjectUtils.createAndGetHomeReposPath();
11631166
Path centralRepositoryDirPath = ballerinaUserHomeDirPath.resolve(ProjectConstants.REPOSITORIES_DIR)
11641167
.resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME);
@@ -1170,7 +1173,7 @@ static boolean pullDependencyPackages(String orgName, String packageName, String
11701173

11711174
ProjectEnvironmentBuilder defaultBuilder = ProjectEnvironmentBuilder.getDefaultBuilder();
11721175
defaultBuilder.addCompilationCacheFactory(new FileSystemCache.FileSystemCacheFactory(cacheDir));
1173-
BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath);
1176+
BalaProject balaProject = BalaProject.loadProject(defaultBuilder, balaPath, buildOptions);
11741177

11751178
// Delete package cache if available
11761179
Path packageCacheDir = cacheDir.resolve(orgName).resolve(packageName).resolve(version);

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/DocCommand.java

+6-2
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,10 @@ public DocCommand() {
114114
"generated by the dependencies")
115115
private Boolean showDependencyDiagnostics;
116116

117+
@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
118+
description = "experimental memory optimization for large projects")
119+
private Boolean optimizeDependencyCompilation;
120+
117121
public void execute() {
118122
if (this.helpFlag) {
119123
String commandUsageInfo = BLauncherCmd.getCommandUsageInfo(DOC_COMMAND);
@@ -226,8 +230,8 @@ private BuildOptions constructBuildOptions() {
226230
.setTestReport(false)
227231
.setObservabilityIncluded(false)
228232
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
229-
.setShowDependencyDiagnostics(showDependencyDiagnostics);
230-
233+
.setShowDependencyDiagnostics(showDependencyDiagnostics)
234+
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);
231235

232236
if (targetDir != null) {
233237
buildOptionsBuilder.targetDir(targetDir.toString());

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PackCommand.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ public class PackCommand implements BLauncherCmd {
9393
"generated by the dependencies")
9494
private Boolean showDependencyDiagnostics;
9595

96+
@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
97+
description = "experimental memory optimization for large projects")
98+
private Boolean optimizeDependencyCompilation;
99+
96100
public PackCommand() {
97101
this.projectPath = Paths.get(System.getProperty(ProjectConstants.USER_DIR));
98102
this.outStream = System.out;
@@ -269,7 +273,8 @@ private BuildOptions constructBuildOptions() {
269273
.setConfigSchemaGen(configSchemaGen)
270274
.setEnableCache(enableCache)
271275
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
272-
.setShowDependencyDiagnostics(showDependencyDiagnostics);
276+
.setShowDependencyDiagnostics(showDependencyDiagnostics)
277+
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);
273278

274279
if (targetDir != null) {
275280
buildOptionsBuilder.targetDir(targetDir.toString());

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/ProfileCommand.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,10 @@ public class ProfileCommand implements BLauncherCmd {
9191
"generated by the dependencies")
9292
private Boolean showDependencyDiagnostics;
9393

94+
@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
95+
description = "experimental memory optimization for large projects")
96+
private Boolean optimizeDependencyCompilation;
97+
9498
private static final String PROFILE_CMD = "bal profile [--debug <port>] [<ballerina-file | package-path>]\n ";
9599

96100
public ProfileCommand() {
@@ -230,7 +234,8 @@ private BuildOptions constructBuildOptions() {
230234
.setTestReport(false)
231235
.setConfigSchemaGen(configSchemaGen)
232236
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
233-
.setShowDependencyDiagnostics(showDependencyDiagnostics);
237+
.setShowDependencyDiagnostics(showDependencyDiagnostics)
238+
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);
234239

235240
if (targetDir != null) {
236241
buildOptionsBuilder.targetDir(targetDir.toString());

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/PullCommand.java

+30-11
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.google.gson.Gson;
2222
import com.google.gson.JsonObject;
2323
import io.ballerina.cli.BLauncherCmd;
24+
import io.ballerina.projects.BuildOptions;
2425
import io.ballerina.projects.JvmTarget;
2526
import io.ballerina.projects.ProjectException;
2627
import io.ballerina.projects.SemanticVersion;
@@ -72,8 +73,9 @@ public class PullCommand implements BLauncherCmd {
7273
private static final String USAGE_TEXT =
7374
"bal pull {<org-name>/<package-name> | <org-name>/<package-name>:<version>}";
7475

75-
private PrintStream errStream;
76-
private boolean exitWhenFinish;
76+
private final PrintStream outStream;
77+
private final PrintStream errStream;
78+
private final boolean exitWhenFinish;
7779

7880
@CommandLine.Parameters
7981
private List<String> argList;
@@ -87,12 +89,20 @@ public class PullCommand implements BLauncherCmd {
8789
@CommandLine.Option(names = "--repository")
8890
private String repositoryName;
8991

92+
@CommandLine.Option(names = "--sticky", hidden = true, defaultValue = "true")
93+
private boolean sticky;
94+
95+
@CommandLine.Option(names = "--offline", hidden = true)
96+
private boolean offline;
97+
9098
public PullCommand() {
99+
this.outStream = System.out;
91100
this.errStream = System.err;
92101
this.exitWhenFinish = true;
93102
}
94103

95104
public PullCommand(PrintStream errStream, boolean exitWhenFinish) {
105+
this.outStream = errStream;
96106
this.errStream = errStream;
97107
this.exitWhenFinish = exitWhenFinish;
98108
}
@@ -254,6 +264,10 @@ public void execute() {
254264
.resolve(ProjectConstants.REPOSITORIES_DIR).resolve(ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME)
255265
.resolve(ProjectConstants.BALA_DIR_NAME)
256266
.resolve(orgName).resolve(packageName);
267+
268+
if (!version.equals(Names.EMPTY.getValue()) && Files.exists(packagePathInBalaCache.resolve(version))) {
269+
outStream.println("Package already exists.\n");
270+
}
257271
// create directory path in bala cache
258272
try {
259273
createDirectories(packagePathInBalaCache);
@@ -267,8 +281,9 @@ public void execute() {
267281
String supportedPlatform = Arrays.stream(JvmTarget.values())
268282
.map(JvmTarget::code)
269283
.collect(Collectors.joining(","));
284+
CentralAPIClient client;
270285
try {
271-
CentralAPIClient client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
286+
client = new CentralAPIClient(RepoUtils.getRemoteRepoURL(),
272287
initializeProxy(settings.getProxy()), settings.getProxy().username(),
273288
settings.getProxy().password(), getAccessTokenOfCLI(settings),
274289
settings.getCentral().getConnectTimeout(),
@@ -281,19 +296,23 @@ public void execute() {
281296
RepoUtils.getBallerinaVersion());
282297
version = CommandUtil.getLatestVersion(versions);
283298
}
284-
boolean hasCompilationErrors = CommandUtil.pullDependencyPackages(orgName, packageName, version);
285-
if (hasCompilationErrors) {
286-
CommandUtil.printError(this.errStream, "compilation contains errors", null, false);
287-
CommandUtil.exitError(this.exitWhenFinish);
288-
return;
289-
}
290299
} catch (PackageAlreadyExistsException e) {
291-
errStream.println(e.getMessage());
292-
CommandUtil.exitError(this.exitWhenFinish);
300+
outStream.println("Package already exists.\n");
301+
version = e.version();
293302
} catch (CentralClientException e) {
294303
errStream.println("package not found: " + orgName + "/" + packageName);
295304
CommandUtil.exitError(this.exitWhenFinish);
296305
}
306+
307+
outStream.println("Resolving dependencies");
308+
BuildOptions buildOptions = BuildOptions.builder().setSticky(sticky).setOffline(offline).build();
309+
boolean hasCompilationErrors = CommandUtil.pullDependencyPackages(orgName, packageName, version, buildOptions);
310+
if (hasCompilationErrors) {
311+
CommandUtil.printError(this.errStream, "compilation contains errors", null, false);
312+
CommandUtil.exitError(this.exitWhenFinish);
313+
return;
314+
}
315+
297316
if (this.exitWhenFinish) {
298317
Runtime.getRuntime().exit(0);
299318
}

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/RunCommand.java

+6-1
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,10 @@ public class RunCommand implements BLauncherCmd {
119119
"generated by the dependencies")
120120
private Boolean showDependencyDiagnostics;
121121

122+
@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
123+
description = "experimental memory optimization for large projects")
124+
private Boolean optimizeDependencyCompilation;
125+
122126
private static final String runCmd =
123127
"""
124128
bal run [--debug <port>] <executable-jar>\s
@@ -289,7 +293,8 @@ private BuildOptions constructBuildOptions() {
289293
.setConfigSchemaGen(configSchemaGen)
290294
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
291295
.setDumpBuildTime(dumpBuildTime)
292-
.setShowDependencyDiagnostics(showDependencyDiagnostics);
296+
.setShowDependencyDiagnostics(showDependencyDiagnostics)
297+
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);
293298

294299
if (targetDir != null) {
295300
buildOptionsBuilder.targetDir(targetDir.toString());

‎cli/ballerina-cli/src/main/java/io/ballerina/cli/cmd/TestCommand.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,9 @@ public TestCommand() {
204204
"generated by the dependencies")
205205
private Boolean showDependencyDiagnostics;
206206

207+
@CommandLine.Option(names = "--optimize-dependency-compilation", hidden = true,
208+
description = "experimental memory optimization for large projects")
209+
private Boolean optimizeDependencyCompilation;
207210

208211
private static final String testCmd = "bal test [--OPTIONS]\n" +
209212
" [<ballerina-file> | <package-path>] [(-Ckey=value)...]";
@@ -389,7 +392,8 @@ private BuildOptions constructBuildOptions() {
389392
.setEnableCache(enableCache)
390393
.disableSyntaxTreeCaching(disableSyntaxTreeCaching)
391394
.setGraalVMBuildOptions(graalVMBuildOptions)
392-
.setShowDependencyDiagnostics(showDependencyDiagnostics);
395+
.setShowDependencyDiagnostics(showDependencyDiagnostics)
396+
.setOptimizeDependencyCompilation(optimizeDependencyCompilation);
393397

394398
if (targetDir != null) {
395399
buildOptionsBuilder.targetDir(targetDir.toString());

‎cli/ballerina-cli/src/main/resources/cli-help/ballerina-build.help

+5
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ OPTIONS
6767
Print the diagnostics that are related to the dependencies. By default, these
6868
diagnostics are not printed to the console.
6969

70+
--optimize-dependency-compilation
71+
[EXPERIMENTAL] Enables memory-efficient compilation of package dependencies
72+
using separate processes. This can help prevent out-of-memory issues during
73+
the initial compilation with a clean central cache.
74+
7075

7176
EXAMPLES
7277
Build the current package. This will generate an 'app.jar' file in the

‎cli/ballerina-cli/src/main/resources/cli-help/ballerina-doc.help

+5
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,11 @@ OPTIONS
3434
Print the diagnostics that are related to the dependencies. By default, these
3535
diagnostics are not printed to the console.
3636

37+
--optimize-dependency-compilation
38+
[EXPERIMENTAL] Enables memory-efficient compilation of package dependencies
39+
using separate processes. This can help prevent out-of-memory issues during
40+
the initial compilation with a clean central cache.
41+
3742

3843
EXAMPLES
3944
Generate API documentation for the current package.

‎cli/ballerina-cli/src/main/resources/cli-help/ballerina-pack.help

+5
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ OPTIONS
2828
Print the diagnostics that are related to the dependencies. By default, these
2929
diagnostics are not printed to the console.
3030

31+
--optimize-dependency-compilation
32+
[EXPERIMENTAL] Enables memory-efficient compilation of package dependencies
33+
using separate processes. This can help prevent out-of-memory issues during
34+
the initial compilation with a clean central cache.
35+
3136

3237
EXAMPLES
3338
Pack the current package into .bala file.

‎cli/ballerina-cli/src/main/resources/cli-help/ballerina-profile.help

+5
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ OPTIONS
2323
Print the diagnostics that are related to the dependencies. By default, these
2424
diagnostics are not printed to the console.
2525

26+
--optimize-dependency-compilation
27+
[EXPERIMENTAL] Enables memory-efficient compilation of package dependencies
28+
using separate processes. This can help prevent out-of-memory issues during
29+
the initial compilation with a clean central cache.
30+
2631
EXAMPLES
2732
Run Ballerina profiler on the 'main' function and service(s) in the 'app.bal' file.
2833
$ bal profile app.bal

‎cli/ballerina-cli/src/main/resources/cli-help/ballerina-run.help

+5
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ OPTIONS
6060
Print the diagnostics that are related to the dependencies. By default, these
6161
diagnostics are not printed to the console.
6262

63+
--optimize-dependency-compilation
64+
[EXPERIMENTAL] Enables memory-efficient compilation of package dependencies
65+
using separate processes. This can help prevent out-of-memory issues during
66+
the initial compilation with a clean central cache.
67+
6368

6469
ARGUMENTS
6570
--

‎cli/ballerina-cli/src/main/resources/cli-help/ballerina-test.help

+5
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@ OPTIONS
105105
Print the diagnostics that are related to the dependencies. By default, these
106106
diagnostics are not printed to the console.
107107

108+
--optimize-dependency-compilation
109+
[EXPERIMENTAL] Enables memory-efficient compilation of package dependencies
110+
using separate processes. This can help prevent out-of-memory issues during
111+
the initial compilation with a clean central cache.
112+
108113

109114
ARGUMENTS
110115
(-Ckey=value)...

‎cli/central-client/src/main/java/org/ballerinalang/central/client/Utils.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -162,12 +162,13 @@ public static void createBalaInHomeRepo(Response balaDownloadResponse, Path pkgP
162162
downloadBody.ifPresent(ResponseBody::close);
163163
throw new PackageAlreadyExistsException(
164164
logFormatter.formatLog("package already exists in the home repository: " +
165-
balaCacheWithPkgPath.toString()));
165+
balaCacheWithPkgPath.toString()), validPkgVersion);
166166
}
167167
} catch (IOException e) {
168168
downloadBody.ifPresent(ResponseBody::close);
169169
throw new PackageAlreadyExistsException(
170-
logFormatter.formatLog("error accessing bala : " + balaCacheWithPkgPath.toString()));
170+
logFormatter.formatLog("error accessing bala : " + balaCacheWithPkgPath.toString()),
171+
validPkgVersion);
171172
}
172173

173174
// Create the following temp path

‎cli/central-client/src/main/java/org/ballerinalang/central/client/exceptions/PackageAlreadyExistsException.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,14 @@
2525
*/
2626
public class PackageAlreadyExistsException extends CentralClientException {
2727

28-
public PackageAlreadyExistsException(String message) {
28+
private final String version;
29+
30+
public PackageAlreadyExistsException(String message, String version) {
2931
super(message);
32+
this.version = version;
33+
}
34+
35+
public String version() {
36+
return version;
3037
}
3138
}

0 commit comments

Comments
 (0)