Fix offline resolution of build tools

cli/ballerina-cli/build.gradle

@@ -121,6 +121,7 @@ test {
     dependsOn copyProfilerResources
 
     systemProperty "ballerina.home", "$buildDir"
+    environment "BALLERINA_HOME_DIR", "${buildDir}/user-home"
 
     useTestNG() {
         suites 'src/test/resources/testng.xml'

cli/ballerina-cli/src/main/java/io/ballerina/cli/launcher/util/BalToolsUtil.java

@@ -90,6 +90,7 @@
 import static io.ballerina.projects.util.ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME;
 import static io.ballerina.projects.util.ProjectConstants.CONFIG_DIR;
 import static io.ballerina.projects.util.ProjectConstants.REPOSITORIES_DIR;
+import static io.ballerina.projects.util.ProjectUtils.BAL_TOOLS_TOML_PATH;
 
 /**
  * This class contains utility functions needed for Bal Tool tasks in the Main class.
@@ -117,8 +118,6 @@
     // if a command is a built-in tool command, add it to this list
     private static final List<String> builtInToolCommands = List.of();
 
-    private static final Path balToolsTomlPath = RepoUtils.createAndGetHomeReposPath().resolve(
-            Path.of(CONFIG_DIR, BAL_TOOLS_TOML));
     private static final Path balaCacheDirPath = ProjectUtils.createAndGetHomeReposPath()
             .resolve(REPOSITORIES_DIR).resolve(CENTRAL_REPOSITORY_CACHE_NAME)
             .resolve(ProjectConstants.BALA_DIR_NAME);
@@ -157,7 +156,7 @@
         if (!builtInToolCommands.contains(commandName)) {
             return;
         }
-        BalToolsToml balToolsToml = BalToolsToml.from(balToolsTomlPath);
+        BalToolsToml balToolsToml = BalToolsToml.from(BAL_TOOLS_TOML_PATH);
         BalToolsManifest balToolsManifest = BalToolsManifestBuilder.from(balToolsToml).build();
         // if built in tool is already added, return
         if (balToolsManifest.getActiveTool(commandName).isPresent()) {
@@ -263,7 +262,7 @@
      * bal-tools.toml file when the user moves from updates 6, 7 to update 8 and above.
      */
     public static void updateOldBalToolsToml() {
-        BalToolsToml balToolsToml = BalToolsToml.from(balToolsTomlPath);
+        BalToolsToml balToolsToml = BalToolsToml.from(BAL_TOOLS_TOML_PATH);
         BalToolsManifestBuilder balToolsManifestBuilder = BalToolsManifestBuilder.from(balToolsToml);
         BalToolsManifest balToolsManifest = balToolsManifestBuilder.build();
         Map<String, BalToolsManifestBuilder.OldTool> oldTools = balToolsManifestBuilder.getOldTools();

cli/ballerina-cli/src/test/java/io/ballerina/cli/cmd/BaseCommandTest.java

@@ -28,17 +28,21 @@
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeSuite;
 import org.testng.annotations.DataProvider;
+import org.wso2.ballerinalang.compiler.util.ProjectDirConstants;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.util.stream.Stream;
 
 import static io.ballerina.projects.util.ProjectConstants.CENTRAL_REPOSITORY_CACHE_NAME;
+import static io.ballerina.projects.util.ProjectConstants.REPOSITORIES_DIR;
 
 /**
  * Command tests super class.
@@ -51,7 +55,10 @@ public abstract class BaseCommandTest {
     protected PrintStream printStream;
     protected Path homeCache;
     private final String userDir = System.getProperty("user.dir");
-
+    final Path testDotBallerina = Paths.get(System.getenv(ProjectDirConstants.HOME_REPO_ENV_KEY));
+    final Path testCentralRepoCache = testDotBallerina.resolve(REPOSITORIES_DIR)
+            .resolve(CENTRAL_REPOSITORY_CACHE_NAME);
+
     @BeforeClass
     public void setup() throws IOException {
         System.setProperty("java.command", "java");
@@ -61,6 +68,16 @@ public void setup() throws IOException {
         this.printStream = new PrintStream(this.console);
     }
 
+    @BeforeSuite
+    public void copyBalTools() throws IOException {
+        // copy the bal-tools.toml to <user-home>/.ballerina/.config/
+        Path balToolsTomlSrcPath = Paths.get("src/test/resources/test-resources/" +
+                "buildToolResources/tools/bal-tools.toml");
+        Path balToolsTomlDstPath = testDotBallerina.resolve(".config");
+        Files.createDirectories(balToolsTomlDstPath);
+        Files.copy(balToolsTomlSrcPath, balToolsTomlDstPath.resolve("bal-tools.toml"));
+    }
+
     @BeforeMethod
     public void beforeMethod() {
         this.console = new ByteArrayOutputStream();

cli/ballerina-cli/src/test/java/io/ballerina/cli/cmd/BuildCommandTest.java

@@ -25,11 +25,8 @@
 import io.ballerina.projects.SemanticVersion;
 import io.ballerina.projects.environment.Environment;
 import io.ballerina.projects.environment.EnvironmentBuilder;
-import io.ballerina.projects.util.BuildToolUtils;
 import io.ballerina.projects.util.ProjectUtils;
 import org.ballerinalang.test.BCompileUtil;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.BeforeGroups;
@@ -640,18 +637,14 @@ public void testBuildEmptyProjectWithCompilerPlugin() throws IOException {
     @Test(description = "Build an empty package with code generator build tools")
     public void testBuildEmptyProjectWithBuildTools() throws IOException {
         BCompileUtil.compileAndCacheBala(testResources.resolve("buildToolResources").resolve("tools")
-                .resolve("ballerina-generate-file").toString(), testDistCacheDirectory, projectEnvironmentBuilder);
+                .resolve("ballerina-generate-file").toString(), testCentralRepoCache, projectEnvironmentBuilder);
         Path projectPath = this.testResources.resolve("emptyProjectWithBuildTool");
         replaceDependenciesTomlContent(projectPath, "**INSERT_DISTRIBUTION_VERSION_HERE**",
                 RepoUtils.getBallerinaShortVersion());
         System.setProperty(USER_DIR_PROPERTY, projectPath.toString());
-        try (MockedStatic<BuildToolUtils> repoUtils = Mockito.mockStatic(
-                BuildToolUtils.class, Mockito.CALLS_REAL_METHODS)) {
-            repoUtils.when(BuildToolUtils::getCentralBalaDirPath).thenReturn(testDistCacheDirectory.resolve("bala"));
-            BuildCommand buildCommand = new BuildCommand(projectPath, printStream, printStream, false);
-            new CommandLine(buildCommand).parseArgs();
-            buildCommand.execute();
-        }
+        BuildCommand buildCommand = new BuildCommand(projectPath, printStream, printStream, false);
+        new CommandLine(buildCommand).parseArgs();
+        buildCommand.execute();
         String buildLog = readOutput(true);
         Assert.assertEquals(buildLog.replace("\r", "").replace("\\", "/"),
                 getOutput("build-empty-project-with-build-tools.txt"));

cli/ballerina-cli/src/test/java/io/ballerina/cli/cmd/RunBuildToolsTaskTest.java

@@ -21,12 +21,11 @@
 import io.ballerina.cli.launcher.BLauncherException;
 import io.ballerina.cli.task.RunBuildToolsTask;
 import io.ballerina.projects.BuildOptions;
+import io.ballerina.projects.JvmTarget;
 import io.ballerina.projects.Project;
 import io.ballerina.projects.directory.BuildProject;
-import io.ballerina.projects.util.BuildToolUtils;
+import io.ballerina.projects.util.ProjectConstants;
 import org.ballerinalang.test.BCompileUtil;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
@@ -42,7 +41,6 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static io.ballerina.cli.cmd.CommandOutputUtils.getOutput;
-import static io.ballerina.projects.util.ProjectConstants.DIST_CACHE_DIRECTORY;
 
 /**
  * Test cases for the RunBuildToolsTask.
@@ -51,9 +49,6 @@
  */
 public class RunBuildToolsTaskTest extends BaseCommandTest {
     private Path buildToolResources;
-    private static final Path testBuildDirectory = Path.of("build").toAbsolutePath();
-    private static final Path testDistCacheDirectory = testBuildDirectory.resolve(DIST_CACHE_DIRECTORY);
-    Path mockCentralBalaDirPath = testDistCacheDirectory.resolve("bala");
 
     private static final long TWO_DAYS = 2 * 24 * 60 * 60 * 1000;
     private static final long HALF_DAY = 12 * 60 * 60 * 1000;
@@ -67,6 +62,7 @@ public void setup() throws IOException {
         super.setup();
         Files.createDirectories(LOG_FILE.getParent());
         Files.writeString(LOG_FILE, "");
+
         // copy all test resources
         try {
             Path testResources = super.tmpDir.resolve("build-tool-test-resources");
@@ -77,19 +73,27 @@ public void setup() throws IOException {
         } catch (Exception e) {
             Assert.fail("error loading resources");
         }
+
         // compile and cache tools
         BCompileUtil.compileAndCacheBala(buildToolResources.resolve("tools")
-                .resolve("dummy-tool-pkg").toString(), testDistCacheDirectory);
+                .resolve("dummy-tool-pkg").toString(), testCentralRepoCache);
         BCompileUtil.compileAndCacheBala(buildToolResources.resolve("tools")
-                        .resolve("invalid-name-tool-pkg").toString(), testDistCacheDirectory);
+                        .resolve("invalid-name-tool-pkg").toString(), testCentralRepoCache);
         BCompileUtil.compileAndCacheBala(buildToolResources.resolve("tools")
-                        .resolve("ballerina-generate-file").toString(), testDistCacheDirectory);
+                        .resolve("ballerina-generate-file").toString(), testCentralRepoCache);
         BCompileUtil.compileAndCacheBala(buildToolResources.resolve("tools")
-                .resolve("hidden-cmd-tool-pkg").toString(), testDistCacheDirectory);
+                .resolve("hidden-cmd-tool-pkg").toString(), testCentralRepoCache);
         BCompileUtil.compileAndCacheBala(buildToolResources.resolve("tools")
-                .resolve("missing-interface-tool-pkg").toString(), testDistCacheDirectory);
+                .resolve("missing-interface-tool-pkg").toString(), testCentralRepoCache);
         BCompileUtil.compileAndCacheBala(buildToolResources.resolve("tools")
-                .resolve("no-options-tool-pkg").toString(), testDistCacheDirectory);
+                .resolve("no-options-tool-pkg").toString(), testCentralRepoCache);
+
+        BCompileUtil.compileAndCacheBala(buildToolResources.resolve("tools")
+                .resolve("dummy-tool-pkg-higher-dist").toString(), testCentralRepoCache);
+
+        Path balaPath = testCentralRepoCache.resolve("bala").resolve("foo")
+                .resolve("dummypkghigherdist").resolve("1.4.0").resolve(JvmTarget.JAVA_21.code());
+        replacePkgDistVersion(balaPath);
         // add build.json files
         addBuildJsonToProjects("project-lt-24h-with-build-tool", System.currentTimeMillis() - HALF_DAY);
         addBuildJsonToProjects("project-gt-24h-with-build-tool", System.currentTimeMillis() - TWO_DAYS);
@@ -98,26 +102,24 @@ public void setup() throws IOException {
     @Test(description = "Resolve a tool offline", dataProvider = "buildToolOfflineProvider")
     public void testOfflineToolResolution(String projectName, String outputFileName, boolean sticky, boolean isError)
             throws IOException {
+
         Path projectPath = buildToolResources.resolve(projectName);
         Project project = BuildProject.load(projectPath,
                 BuildOptions.builder().setOffline(true).setSticky(sticky).build());
         RunBuildToolsTask runBuildToolsTask = new RunBuildToolsTask(printStream);
-        try (MockedStatic<BuildToolUtils> repoUtils = Mockito.mockStatic(
-                BuildToolUtils.class, Mockito.CALLS_REAL_METHODS)) {
-            repoUtils.when(BuildToolUtils::getCentralBalaDirPath).thenReturn(mockCentralBalaDirPath);
-            try {
-                runBuildToolsTask.execute(project);
-            } catch (BLauncherException e) {
-                if (!isError) {
-                    String errorMsg = "Error executing build tools task for project: " + projectName
-                            + (sticky ? "with sticky." : "without sticky. ") + e.getMessage();
-                    Assert.fail(errorMsg);
-                }
-                List<String> messages = e.getMessages();
-                Assert.assertEquals(messages.size(), 1);
-                Assert.assertEquals(messages.get(0), "error: build tool execution contains errors");
+        try {
+            runBuildToolsTask.execute(project);
+        } catch (BLauncherException e) {
+            if (!isError) {
+                String errorMsg = "Error executing build tools task for project: " + projectName
+                        + (sticky ? " with sticky." : " without sticky. ") + e.getMessage();
+                Assert.fail(errorMsg);
             }
+            List<String> messages = e.getMessages();
+            Assert.assertEquals(messages.size(), 1);
+            Assert.assertEquals(messages.get(0), "error: build tool execution contains errors");
         }
+
         String buildLog = readOutput(true);
         Assert.assertEquals(buildLog.replace("\r", ""), getOutput(outputFileName));
     }
@@ -127,11 +129,7 @@ public void testProjectForAddedGeneratedCode() throws IOException {
         Path projectPath = buildToolResources.resolve("project-with-generate-file-tool");
         Project project = BuildProject.load(projectPath, BuildOptions.builder().setOffline(true).build());
         RunBuildToolsTask runBuildToolsTask = new RunBuildToolsTask(printStream);
-        try (MockedStatic<BuildToolUtils> repoUtils = Mockito.mockStatic(
-                BuildToolUtils.class, Mockito.CALLS_REAL_METHODS)) {
-            repoUtils.when(BuildToolUtils::getCentralBalaDirPath).thenReturn(mockCentralBalaDirPath);
-            runBuildToolsTask.execute(project);
-        }
+        runBuildToolsTask.execute(project);
         String buildLog = readOutput(true);
         Assert.assertEquals(buildLog.replace("\r", ""), getOutput("build-tool-generate-file.txt"));
         AtomicBoolean fileFound = new AtomicBoolean(false);
@@ -241,6 +239,12 @@ public Object[][] buildToolOfflineProvider() {
                 false,
                 false
             },
+            {
+                "project-with-higher-dist-build-tool",
+                "build-tool-higher-dist-resolve-failed.txt",
+                false,
+                true
+            }
         };
     }
 
@@ -268,4 +272,15 @@ public void cleanUp() throws IOException {
         Files.deleteIfExists(LOG_FILE.getParent());
         Files.deleteIfExists(LOG_FILE.getParent().getParent());
     }
+
+    private void replacePkgDistVersion(Path balaPath) {
+        Path packageJson = balaPath.resolve(ProjectConstants.PACKAGE_JSON);
+        try {
+            String content = Files.readString(packageJson);
+            content = content.replace(RepoUtils.getBallerinaShortVersion(), "2201.99.0");
+            Files.writeString(packageJson, content);
+        } catch (IOException e) {
+            Assert.fail("Error replacing distribution version in bala");
+        }
+    }
 }

cli/ballerina-cli/src/test/java/io/ballerina/cli/cmd/RunCommandTest.java

@@ -5,12 +5,9 @@
 import io.ballerina.projects.ProjectEnvironmentBuilder;
 import io.ballerina.projects.environment.Environment;
 import io.ballerina.projects.environment.EnvironmentBuilder;
-import io.ballerina.projects.util.BuildToolUtils;
 import io.ballerina.projects.util.ProjectUtils;
 import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.ballerinalang.test.BCompileUtil;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
 import org.testng.Assert;
 import org.testng.annotations.AfterSuite;
 import org.testng.annotations.BeforeClass;
@@ -415,18 +412,14 @@ public void testRunEmptyPackage() throws IOException {
     public void testRunEmptyProjectWithBuildTools() throws IOException {
         BCompileUtil.compileAndCacheBala(
                 testResources.resolve("buildToolResources/tools/ballerina-generate-file").toString(),
-                testDistCacheDirectory, projectEnvironmentBuilder);
+                testCentralRepoCache, projectEnvironmentBuilder);
         Path projectPath = this.testResources.resolve("emptyProjectWithBuildTool");
         replaceDependenciesTomlContent(projectPath, "**INSERT_DISTRIBUTION_VERSION_HERE**",
                 RepoUtils.getBallerinaShortVersion());
         System.setProperty(USER_DIR_PROPERTY, projectPath.toString());
-        try (MockedStatic<BuildToolUtils> repoUtils = Mockito.mockStatic(
-                BuildToolUtils.class, Mockito.CALLS_REAL_METHODS)) {
-            repoUtils.when(BuildToolUtils::getCentralBalaDirPath).thenReturn(testDistCacheDirectory.resolve("bala"));
-            RunCommand runCommand = new RunCommand(projectPath, printStream, false);
-            new CommandLine(runCommand).parseArgs();
-            runCommand.execute();
-        }
+        RunCommand runCommand = new RunCommand(projectPath, printStream, false);
+        new CommandLine(runCommand).parseArgs();
+        runCommand.execute();
         String buildLog = readOutput(true);
         Assert.assertEquals(buildLog.replaceAll("\r", ""), getOutput("run-empty-project-with-build-tools.txt"));
     }

cli/ballerina-cli/src/test/java/io/ballerina/cli/cmd/TestCommandTest.java

@@ -23,7 +23,6 @@
 import io.ballerina.projects.ProjectEnvironmentBuilder;
 import io.ballerina.projects.environment.Environment;
 import io.ballerina.projects.environment.EnvironmentBuilder;
-import io.ballerina.projects.util.BuildToolUtils;
 import io.ballerina.projects.util.ProjectConstants;
 import io.ballerina.projects.util.ProjectUtils;
 import org.apache.commons.codec.digest.DigestUtils;
@@ -417,18 +416,14 @@ public void testTestEmptyPackage() throws IOException {
     public void testTestEmptyProjectWithBuildTools() throws IOException {
         BCompileUtil.compileAndCacheBala(
                 testResources.resolve("buildToolResources/tools/ballerina-generate-file").toString(),
-                testDistCacheDirectory, projectEnvironmentBuilder);
+                testCentralRepoCache, projectEnvironmentBuilder);
         Path projectPath = this.testResources.resolve("emptyProjectWithBuildTool");
         replaceDependenciesTomlContent(projectPath, "**INSERT_DISTRIBUTION_VERSION_HERE**",
                 RepoUtils.getBallerinaShortVersion());
         System.setProperty("user.dir", projectPath.toString());
-        try (MockedStatic<BuildToolUtils> repoUtils = Mockito.mockStatic(
-                BuildToolUtils.class, Mockito.CALLS_REAL_METHODS)) {
-            repoUtils.when(BuildToolUtils::getCentralBalaDirPath).thenReturn(testDistCacheDirectory.resolve("bala"));
-            TestCommand testCommand = new TestCommand(projectPath, printStream, printStream, false);
-            new CommandLine(testCommand).parseArgs();
-            testCommand.execute();
-        }
+        TestCommand testCommand = new TestCommand(projectPath, printStream, printStream, false);
+        new CommandLine(testCommand).parseArgs();
+        testCommand.execute();
         String buildLog = readOutput(true);
         Assert.assertEquals(buildLog.replace("\r", ""), getOutput("test-empty-project-with-build-tools.txt"));
     }

cli/ballerina-cli/src/test/resources/test-resources/buildToolResources/fresh-project-with-central-build-tool/Ballerina.toml

@@ -3,7 +3,7 @@ org = "foo"
 name = "winery"
 version = "0.1.0"
 
-[[tool.dummy_tool]]
+[[tool.dummy_tool2]]
 id = "main_dummy"
 filePath = "delivery.json"
 targetModule = "mod_main"

cli/ballerina-cli/src/test/resources/test-resources/buildToolResources/project-with-higher-dist-build-tool/Ballerina.toml

@@ -0,0 +1,10 @@
+[package]
+org = "foo"
+name = "winery"
+version = "0.1.0"
+
+[[tool.dummy_tool_higher_dist]]
+id = "main_dummy"
+filePath = "delivery.json"
+targetModule = "mod_main"
+options.mode = "client"

cli/ballerina-cli/src/test/resources/test-resources/buildToolResources/project-with-higher-dist-build-tool/Dependencies.toml

@@ -0,0 +1,23 @@
+	# AUTO-GENERATED FILE. DO NOT MODIFY.
+
+# This file is auto-generated by Ballerina for managing dependency versions.
+# It should not be modified by hand.
+
+[ballerina]
+dependencies-toml-version = "2"
+distribution-version = "2201.8.5"
+
+[[package]]
+org = "foo"
+name = "winery"
+version = "0.1.0"
+modules = [
+	{org = "foo", packageName = "winery", moduleName = "winery"}
+]
+
+[[tool]]
+id = "dummy_tool"
+org = "foo"
+name = "dummypkg"
+version = "1.4.0"
+

cli/ballerina-cli/src/test/resources/test-resources/buildToolResources/project-with-higher-dist-build-tool/main.bal

@@ -0,0 +1,3 @@
+public function main() {
+    return;
+}