diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/generator/compiler/PureCompilerBinaryGenerator.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/generator/compiler/PureCompilerBinaryGenerator.java index d600ee4d3f..4b16087663 100644 --- a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/generator/compiler/PureCompilerBinaryGenerator.java +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/generator/compiler/PureCompilerBinaryGenerator.java @@ -16,6 +16,7 @@ import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.list.ListIterable; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.api.set.MutableSet; import org.eclipse.collections.api.set.SetIterable; @@ -38,11 +39,17 @@ import org.finos.legend.pure.m3.serialization.runtime.PureCompilerLoader; import org.finos.legend.pure.m3.serialization.runtime.PureRuntime; import org.finos.legend.pure.m3.serialization.runtime.PureRuntimeBuilder; +import org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator; import org.finos.legend.pure.m4.exception.PureCompilationException; import org.finos.legend.pure.m4.serialization.grammar.antlr.PureParserException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Set; @@ -65,10 +72,20 @@ public static void main(String[] args) public static void serializeModules(Path outputDirectory, Iterable modules) { - serializeModules(outputDirectory, null, modules, null); + serializeModules(outputDirectory, modules, modules == null); + } + + public static void serializeModules(Path outputDirectory, Iterable modules, boolean serializeIndividually) + { + serializeModules(outputDirectory, null, modules, null, serializeIndividually); } public static void serializeModules(Path outputDirectory, ClassLoader classLoader, Iterable modules, Iterable excludedModules) + { + serializeModules(outputDirectory, classLoader, modules, excludedModules, modules == null); + } + + public static void serializeModules(Path outputDirectory, ClassLoader classLoader, Iterable modules, Iterable excludedModules, boolean serializeIndividually) { long start = System.nanoTime(); SetIterable moduleSet = (modules == null) ? Sets.immutable.empty() : @@ -87,9 +104,7 @@ public static void serializeModules(Path outputDirectory, ClassLoader classLoade try { FilePathProvider filePathProvider = FilePathProvider.builder().withLoadedExtensions(currentClassLoader).build(); - RepositoryInfo repositoryInfo = resolveRepositories(moduleSet, excludedModules, currentClassLoader, filePathProvider); - PureRuntime runtime = compile(currentClassLoader, repositoryInfo.toCompile); - serialize(outputDirectory, repositoryInfo.toSerialize, runtime, filePathProvider); + serializeModules(outputDirectory, currentClassLoader, moduleSet, excludedModules, filePathProvider, serializeIndividually); } catch (Throwable t) { @@ -107,18 +122,22 @@ public static void serializeModules(Path outputDirectory, ClassLoader classLoade } } - private static RepositoryInfo resolveRepositories(SetIterable modules, Iterable excludedModules, ClassLoader classLoader, FilePathProvider filePathProvider) + private static void serializeModules(Path outputDirectory, ClassLoader classLoader, SetIterable modules, Iterable excludedModules, FilePathProvider filePathProvider, boolean serializeIndividually) { - MutableList foundRepos = CodeRepositoryProviderHelper.findCodeRepositories(true).toList(); + // Build the full repository set (with exclusions applied) + MutableList foundRepos = CodeRepositoryProviderHelper.findCodeRepositories(classLoader, true).toList(); LOGGER.debug("Found repositories: {}", foundRepos.asLazy().collect(CodeRepository::getName)); - CodeRepositorySet.Builder builder = CodeRepositorySet.builder().withCodeRepositories(foundRepos); + CodeRepositorySet.Builder fullSetBuilder = CodeRepositorySet.builder().withCodeRepositories(foundRepos); if (excludedModules != null) { - builder.withoutCodeRepositories(excludedModules); + fullSetBuilder.withoutCodeRepositories(excludedModules); } + + // Determine which modules to serialize SetIterable toSerialize; if (modules.isEmpty()) { + // Serialize all modules that don't already have a manifest on the classpath SetIterable excludedSet = (excludedModules == null) ? Sets.immutable.empty() : Sets.mutable.withAll(excludedModules); toSerialize = foundRepos.collectIf( repo -> !excludedSet.contains(repo.getName()) && classLoader.getResource(filePathProvider.getModuleManifestResourceName(repo.getName())) == null, @@ -132,14 +151,73 @@ private static RepositoryInfo resolveRepositories(SetIterable modules, I } if (toSerialize.notEmpty()) { - builder.subset(toSerialize); + fullSetBuilder.subset(toSerialize); + } + CodeRepositorySet allRepositories = fullSetBuilder.build(); + + if (serializeIndividually) + { + // Order the modules to serialize by dependency (dependencies first) + ListIterable orderedModules; + if (toSerialize.size() == 1) + { + orderedModules = Lists.immutable.with(toSerialize.getAny()); + } + else + { + orderedModules = toSerialize.isEmpty() ? + CodeRepository.toSortedRepositoryList(allRepositories.getRepositories()).collect(CodeRepository::getName) : + toSerialize.toSortedList(new RepositoryComparator(allRepositories.getRepositories())); + LOGGER.debug("Serializing modules in order: {}", orderedModules); + } + + // Create a class loader that includes the output directory so that previously serialized modules can be loaded + try (URLClassLoader outputClassLoader = newClassLoaderWithOutputDirectory(classLoader, outputDirectory)) + { + PureCompilerLoader loader = PureCompilerLoader.newLoader(outputClassLoader); + orderedModules.forEach(m -> serializeModules(outputDirectory, classLoader, Sets.immutable.with(m), allRepositories, loader, filePathProvider)); + } + catch (IOException e) + { + throw new UncheckedIOException("Error closing class loader", e); + } + } + else + { + serializeModules(outputDirectory, classLoader, toSerialize, allRepositories, PureCompilerLoader.newLoader(classLoader), filePathProvider); } - CodeRepositorySet resolvedRepositories = builder.build(); - LOGGER.debug("Resolved repositories: {}", resolvedRepositories.getRepositoryNames()); - return new RepositoryInfo(resolvedRepositories, toSerialize); } - private static PureRuntime compile(ClassLoader classLoader, CodeRepositorySet codeRepositories) + private static void serializeModules(Path outputDirectory, ClassLoader classLoader, SetIterable modulesToSerialize, CodeRepositorySet modulesToCompile, PureCompilerLoader loader, FilePathProvider filePathProvider) + { + long moduleStart = System.nanoTime(); + LOGGER.info("Starting compilation and serialization of {}", (modulesToSerialize.size() == 1) ? modulesToSerialize.getAny() : modulesToSerialize); + try + { + LOGGER.debug("Compiling modules {}", modulesToCompile.getRepositoryNames()); + PureRuntime runtime = compile(classLoader, loader, modulesToCompile); + serialize(outputDirectory, modulesToSerialize, runtime, filePathProvider); + } + finally + { + long moduleEnd = System.nanoTime(); + LOGGER.info("Finished compilation and serialization of {} in {}s", (modulesToSerialize.size() == 1) ? modulesToSerialize.getAny() : modulesToSerialize, (moduleEnd - moduleStart) / 1_000_000_000.0); + } + } + + private static URLClassLoader newClassLoaderWithOutputDirectory(ClassLoader parent, Path outputDirectory) + { + try + { + return new URLClassLoader(new URL[]{outputDirectory.toUri().toURL()}, parent); + } + catch (MalformedURLException e) + { + throw new RuntimeException("Error creating class loader with output directory: " + outputDirectory, e); + } + } + + private static PureRuntime compile(ClassLoader classLoader, PureCompilerLoader loader, CodeRepositorySet codeRepositories) { long start = System.nanoTime(); LOGGER.info("Starting compilation"); @@ -150,7 +228,6 @@ private static PureRuntime compile(ClassLoader classLoader, CodeRepositorySet co .setTransactionalByDefault(false) .build(); - PureCompilerLoader loader = PureCompilerLoader.newLoader(classLoader); MutableList reposToLoad = Lists.mutable.empty(); MutableList reposToCompile = Lists.mutable.empty(); codeRepositories.getRepositoryNames().forEach(r -> (loader.canLoad(r) ? reposToLoad : reposToCompile).add(r)); @@ -264,16 +341,4 @@ private static void serialize(Path outputDirectory, SetIterable modules, LOGGER.info("Finished serialization in {}s", (end - start) / 1_000_000_000.0); } } - - private static class RepositoryInfo - { - private final CodeRepositorySet toCompile; - private final SetIterable toSerialize; - - private RepositoryInfo(CodeRepositorySet toCompile, SetIterable toSerialize) - { - this.toCompile = toCompile; - this.toSerialize = toSerialize; - } - } } diff --git a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/file/FileDeserializer.java b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/file/FileDeserializer.java index 3c4d08a072..f9a3412eb2 100644 --- a/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/file/FileDeserializer.java +++ b/legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/file/FileDeserializer.java @@ -1121,7 +1121,7 @@ public Builder withSerializers(ConcreteElementDeserializer elementDeserializer, public FileDeserializer build() { Objects.requireNonNull(this.filePathProvider, "file path provider is required"); - Objects.requireNonNull(this.elementDeserializer, "concrete element serializer is required"); + Objects.requireNonNull(this.elementDeserializer, "concrete element deserializer is required"); Objects.requireNonNull(this.moduleSerializer, "module serializer is required"); return new FileDeserializer(this.filePathProvider, this.elementDeserializer, this.moduleSerializer); } diff --git a/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/generator/compiler/TestPureCompilerBinaryGenerator.java b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/generator/compiler/TestPureCompilerBinaryGenerator.java new file mode 100644 index 0000000000..7f6df3448d --- /dev/null +++ b/legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/generator/compiler/TestPureCompilerBinaryGenerator.java @@ -0,0 +1,186 @@ +// Copyright 2026 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.m3.generator.compiler; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.set.MutableSet; +import org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementDeserializer; +import org.finos.legend.pure.m3.serialization.compiler.file.FileDeserializer; +import org.finos.legend.pure.m3.serialization.compiler.file.FilePathProvider; +import org.finos.legend.pure.m3.serialization.compiler.metadata.ModuleManifest; +import org.finos.legend.pure.m3.serialization.compiler.metadata.ModuleMetadataSerializer; +import org.junit.Assert; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.stream.Stream; + +public class TestPureCompilerBinaryGenerator +{ + @ClassRule + public static TemporaryFolder TMP = new TemporaryFolder(); + + private static final String PLATFORM = "platform"; + private static final String TEST_REPO = "test_generic_repository"; + private static final String OTHER_TEST_REPO = "other_test_generic_repository"; + + @Test + public void testSerializeSingleModule() throws IOException + { + Path outputDirectory = TMP.newFolder().toPath(); + PureCompilerBinaryGenerator.serializeModules(outputDirectory, Lists.immutable.with(TEST_REPO)); + + FileDeserializer deserializer = newFileDeserializer(); + assertModuleSerialized(deserializer, outputDirectory, TEST_REPO); + assertModuleNotSerialized(deserializer, outputDirectory, PLATFORM); + assertModuleNotSerialized(deserializer, outputDirectory, OTHER_TEST_REPO); + } + + @Test + public void testSerializeAllModules() throws IOException + { + FileDeserializer deserializer = newFileDeserializer(); + + // Check the case where all modules are implicitly specified and serialized individually + Path implicitOutputDir = TMP.newFolder().toPath(); + PureCompilerBinaryGenerator.serializeModules(implicitOutputDir, null, true); + + assertModuleSerialized(deserializer, implicitOutputDir, PLATFORM); + assertModuleSerialized(deserializer, implicitOutputDir, TEST_REPO); + assertModuleSerialized(deserializer, implicitOutputDir, OTHER_TEST_REPO); + + // Check the case where all modules are explicitly specified and serialized together + Path explicitOutputDir = TMP.newFolder().toPath(); + PureCompilerBinaryGenerator.serializeModules(explicitOutputDir, Lists.immutable.with(PLATFORM, TEST_REPO, OTHER_TEST_REPO), false); + + assertModuleSerialized(deserializer, explicitOutputDir, PLATFORM); + assertModuleSerialized(deserializer, explicitOutputDir, TEST_REPO); + assertModuleSerialized(deserializer, explicitOutputDir, OTHER_TEST_REPO); + + // Assert that the directories have identical content + assertDirectoriesEquivalent(implicitOutputDir, explicitOutputDir); + } + + @Test + public void testSerializeMultipleButNotAllModules() throws IOException + { + Path outputDirectory = TMP.newFolder().toPath(); + PureCompilerBinaryGenerator.serializeModules(outputDirectory, Lists.immutable.with(TEST_REPO, OTHER_TEST_REPO)); + + FileDeserializer deserializer = newFileDeserializer(); + assertModuleSerialized(deserializer, outputDirectory, TEST_REPO); + assertModuleSerialized(deserializer, outputDirectory, OTHER_TEST_REPO); + assertModuleNotSerialized(deserializer, outputDirectory, PLATFORM); + } + + @Test + public void testSerializeWithExcludedModule() throws IOException + { + Path outputDirectory = TMP.newFolder().toPath(); + PureCompilerBinaryGenerator.serializeModules(outputDirectory, null, null, Lists.immutable.with(OTHER_TEST_REPO)); + + FileDeserializer deserializer = newFileDeserializer(); + assertModuleSerialized(deserializer, outputDirectory, PLATFORM); + assertModuleSerialized(deserializer, outputDirectory, TEST_REPO); + assertModuleNotSerialized(deserializer, outputDirectory, OTHER_TEST_REPO); + } + + private static void assertModuleSerialized(FileDeserializer deserializer, Path outputDirectory, String moduleName) + { + Assert.assertTrue(moduleName + " manifest should exist", deserializer.moduleManifestExists(outputDirectory, moduleName)); + ModuleManifest manifest = deserializer.deserializeModuleManifest(outputDirectory, moduleName); + Assert.assertEquals(moduleName, manifest.getModuleName()); + manifest.forEachElement(element -> + { + String elementPath = element.getPath(); + Assert.assertTrue(moduleName + " / " + elementPath, deserializer.elementExists(outputDirectory, elementPath)); + }); + } + + private static void assertModuleNotSerialized(FileDeserializer deserializer, Path outputDirectory, String moduleName) + { + Assert.assertFalse(moduleName + " manifest should not exist", deserializer.moduleManifestExists(outputDirectory, moduleName)); + } + + private static void assertDirectoriesEquivalent(Path dir1, Path dir2) + { + // First check that relative files paths are the same + MutableSet paths1 = Sets.mutable.empty(); + MutableSet paths2 = Sets.mutable.empty(); + try (Stream stream1 = Files.walk(dir1); + Stream stream2 = Files.walk(dir2)) + { + stream1.map(dir1::relativize).forEach(paths1::add); + stream2.map(dir2::relativize).forEach(paths2::add); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + Assert.assertEquals(paths1, paths2); + + MutableList mismatchFiles = Lists.mutable.empty(); + paths1.forEach(path -> + { + Path file1 = dir1.resolve(path); + Path file2 = dir2.resolve(path); + if (Files.isDirectory(file1)) + { + if (!Files.isDirectory(file2)) + { + mismatchFiles.add(path); + } + } + else if (Files.isDirectory(file2)) + { + mismatchFiles.add(path); + } + else + { + try + { + byte[] bytes1 = Files.readAllBytes(file1); + byte[] bytes2 = Files.readAllBytes(file2); + if (!Arrays.equals(bytes1, bytes2)) + { + mismatchFiles.add(path); + } + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + }); + Assert.assertEquals(Lists.fixedSize.empty(), mismatchFiles); + } + + private static FileDeserializer newFileDeserializer() + { + return FileDeserializer.builder() + .withFilePathProvider(FilePathProvider.builder().withLoadedExtensions().build()) + .withConcreteElementDeserializer(ConcreteElementDeserializer.builder().withLoadedExtensions().build()) + .withModuleMetadataSerializer(ModuleMetadataSerializer.builder().withLoadedExtensions().build()) + .build(); + } +} diff --git a/legend-pure-maven/legend-pure-maven-compiler/src/main/java/org/finos/legend/pure/maven/compiler/PureCompilerMojo.java b/legend-pure-maven/legend-pure-maven-compiler/src/main/java/org/finos/legend/pure/maven/compiler/PureCompilerMojo.java index d5179df6e8..a8564df9b0 100644 --- a/legend-pure-maven/legend-pure-maven-compiler/src/main/java/org/finos/legend/pure/maven/compiler/PureCompilerMojo.java +++ b/legend-pure-maven/legend-pure-maven-compiler/src/main/java/org/finos/legend/pure/maven/compiler/PureCompilerMojo.java @@ -63,6 +63,18 @@ public class PureCompilerMojo extends AbstractMojo @Parameter private Set excludedRepositories; + /** + *

If there are multiple repositories, whether to compile them individually or all together.

+ * + *

If multiple repositories are compiled individually, they will be ordered topologically based on dependencies. + * This means that all of a repository's dependencies will be compiled before it is.

+ * + *

The default value depends on how repositories are specified: if repositories are explicitly specified, the + * default is false (compile them all together); otherwise, the default is true (compile them individually).

+ */ + @Parameter + private Boolean compileIndividually; + /** *

The scope of the dependencies to resolve from the Maven module. Use names from {@link DependencyResolutionScope}. * If not specified, defaults to @@ -98,37 +110,39 @@ public class PureCompilerMojo extends AbstractMojo @Override public void execute() throws MojoExecutionException, MojoFailureException { - ClassLoader savedClassLoader = Thread.currentThread().getContextClassLoader(); + DependencyResolutionScope dependencyResolutionScope = ProjectDependencyResolution.determineDependencyResolutionScope(this.dependencyScope, this.mojoExecution); + URL[] dependencyUrls; try { - DependencyResolutionScope dependencyResolutionScope = ProjectDependencyResolution.determineDependencyResolutionScope(dependencyScope, mojoExecution); - URL[] dependencyUrls = ProjectDependencyResolution.getDependencyURLs( + dependencyUrls = ProjectDependencyResolution.getDependencyURLs( dependencyResolutionScope, - mavenProject, - mojoExecution, - mavenRepoSession, - projectOutputDirectory, - projectTestOutputDirectory, - mavenProjectDependenciesResolver + this.mavenProject, + this.mojoExecution, + this.mavenRepoSession, + this.projectOutputDirectory, + this.projectTestOutputDirectory, + this.mavenProjectDependenciesResolver ); - try (URLClassLoader classLoader = new URLClassLoader(dependencyUrls, Thread.currentThread().getContextClassLoader())) - { - Thread.currentThread().setContextClassLoader(classLoader); - this.executeWithinClassLoader(classLoader, dependencyResolutionScope); - } - catch (IOException e) - { - throw new MojoExecutionException("Error closing classloader", e); - } - finally - { - Thread.currentThread().setContextClassLoader(savedClassLoader); - } } catch (DependencyResolutionException e) { throw new MojoExecutionException("Error setting up classloader with project dependencies", e); } + + ClassLoader savedClassLoader = Thread.currentThread().getContextClassLoader(); + try (URLClassLoader classLoader = new URLClassLoader(dependencyUrls, savedClassLoader)) + { + Thread.currentThread().setContextClassLoader(classLoader); + executeWithinClassLoader(classLoader, dependencyResolutionScope); + } + catch (IOException e) + { + throw new MojoExecutionException("Error closing classloader", e); + } + finally + { + Thread.currentThread().setContextClassLoader(savedClassLoader); + } } private void executeWithinClassLoader(ClassLoader classLoader, DependencyResolutionScope dependencyResolutionScope) throws MojoExecutionException, MojoFailureException @@ -148,9 +162,12 @@ private void executeWithinClassLoader(ClassLoader classLoader, DependencyResolut } getLog().debug("Resolved repositories: " + ((resolvedRepos == null) ? "" : String.join(", ", resolvedRepos))); + boolean serializeReposIndividually = shouldSerializeIndividually(resolvedRepos); + getLog().debug("Compiling repositories individually: " + this.compileIndividually); + try { - PureCompilerBinaryGenerator.serializeModules(resolvedOutputDir.toPath(), classLoader, resolvedRepos, this.excludedRepositories); + PureCompilerBinaryGenerator.serializeModules(resolvedOutputDir.toPath(), classLoader, resolvedRepos, this.excludedRepositories, serializeReposIndividually); } catch (PureCompilationException | PureParserException e) { @@ -217,6 +234,18 @@ private Set resolveRepositoriesToSerialize(DependencyResolutionScope res return foundRepositories; } + private boolean shouldSerializeIndividually(Set resolvedRepos) + { + // If the user has specified whether to serialize individually, use that + if (this.compileIndividually != null) + { + return this.compileIndividually; + } + + // If the user has specified repos to serialize, serialize them together; otherwise, serialize individually + return !isNonEmpty(resolvedRepos); + } + private void forEachRepoDefinition(File directory, Consumer consumer) { forEachRepoDefinition(directory.toPath(), consumer);