Allow serializing multiple modules individually or together

legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/generator/compiler/PureCompilerBinaryGenerator.java

@@ -16,6 +16,7 @@
 
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.factory.Sets;
+import org.eclipse.collections.api.list.ListIterable;
 import org.eclipse.collections.api.list.MutableList;
 import org.eclipse.collections.api.set.MutableSet;
 import org.eclipse.collections.api.set.SetIterable;
@@ -38,11 +39,17 @@
 import org.finos.legend.pure.m3.serialization.runtime.PureCompilerLoader;
 import org.finos.legend.pure.m3.serialization.runtime.PureRuntime;
 import org.finos.legend.pure.m3.serialization.runtime.PureRuntimeBuilder;
+import org.finos.legend.pure.m3.serialization.runtime.RepositoryComparator;
 import org.finos.legend.pure.m4.exception.PureCompilationException;
 import org.finos.legend.pure.m4.serialization.grammar.antlr.PureParserException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Set;
@@ -65,10 +72,20 @@ public static void main(String[] args)
 
     public static void serializeModules(Path outputDirectory, Iterable<String> modules)
     {
-        serializeModules(outputDirectory, null, modules, null);
+        serializeModules(outputDirectory, modules, modules == null);
+    }
+
+    public static void serializeModules(Path outputDirectory, Iterable<String> modules, boolean serializeIndividually)
+    {
+        serializeModules(outputDirectory, null, modules, null, serializeIndividually);
     }
 
     public static void serializeModules(Path outputDirectory, ClassLoader classLoader, Iterable<String> modules, Iterable<String> excludedModules)
+    {
+        serializeModules(outputDirectory, classLoader, modules, excludedModules, modules == null);
+    }
+
+    public static void serializeModules(Path outputDirectory, ClassLoader classLoader, Iterable<String> modules, Iterable<String> excludedModules, boolean serializeIndividually)
     {
         long start = System.nanoTime();
         SetIterable<String> moduleSet = (modules == null) ? Sets.immutable.empty() :
@@ -87,9 +104,7 @@ public static void serializeModules(Path outputDirectory, ClassLoader classLoade
         try
         {
             FilePathProvider filePathProvider = FilePathProvider.builder().withLoadedExtensions(currentClassLoader).build();
-            RepositoryInfo repositoryInfo = resolveRepositories(moduleSet, excludedModules, currentClassLoader, filePathProvider);
-            PureRuntime runtime = compile(currentClassLoader, repositoryInfo.toCompile);
-            serialize(outputDirectory, repositoryInfo.toSerialize, runtime, filePathProvider);
+            serializeModules(outputDirectory, currentClassLoader, moduleSet, excludedModules, filePathProvider, serializeIndividually);
         }
         catch (Throwable t)
         {
@@ -107,18 +122,22 @@ public static void serializeModules(Path outputDirectory, ClassLoader classLoade
         }
     }
 
-    private static RepositoryInfo resolveRepositories(SetIterable<String> modules, Iterable<String> excludedModules, ClassLoader classLoader, FilePathProvider filePathProvider)
+    private static void serializeModules(Path outputDirectory, ClassLoader classLoader, SetIterable<String> modules, Iterable<String> excludedModules, FilePathProvider filePathProvider, boolean serializeIndividually)
     {
-        MutableList<CodeRepository> foundRepos = CodeRepositoryProviderHelper.findCodeRepositories(true).toList();
+        // Build the full repository set (with exclusions applied)
+        MutableList<CodeRepository> foundRepos = CodeRepositoryProviderHelper.findCodeRepositories(classLoader, true).toList();
         LOGGER.debug("Found repositories: {}", foundRepos.asLazy().collect(CodeRepository::getName));
-        CodeRepositorySet.Builder builder = CodeRepositorySet.builder().withCodeRepositories(foundRepos);
+        CodeRepositorySet.Builder fullSetBuilder = CodeRepositorySet.builder().withCodeRepositories(foundRepos);
         if (excludedModules != null)
         {
-            builder.withoutCodeRepositories(excludedModules);
+            fullSetBuilder.withoutCodeRepositories(excludedModules);
         }
+
+        // Determine which modules to serialize
         SetIterable<String> toSerialize;
         if (modules.isEmpty())
         {
+            // Serialize all modules that don't already have a manifest on the classpath
             SetIterable<String> excludedSet = (excludedModules == null) ? Sets.immutable.empty() : Sets.mutable.withAll(excludedModules);
             toSerialize = foundRepos.collectIf(
                     repo -> !excludedSet.contains(repo.getName()) && classLoader.getResource(filePathProvider.getModuleManifestResourceName(repo.getName())) == null,
@@ -132,14 +151,73 @@ private static RepositoryInfo resolveRepositories(SetIterable<String> modules, I
         }
         if (toSerialize.notEmpty())
         {
-            builder.subset(toSerialize);
+            fullSetBuilder.subset(toSerialize);
+        }
+        CodeRepositorySet allRepositories = fullSetBuilder.build();
+
+        if (serializeIndividually)
+        {
+            // Order the modules to serialize by dependency (dependencies first)
+            ListIterable<String> orderedModules;
+            if (toSerialize.size() == 1)
+            {
+                orderedModules = Lists.immutable.with(toSerialize.getAny());
+            }
+            else
+            {
+                orderedModules = toSerialize.isEmpty() ?
+                                 CodeRepository.toSortedRepositoryList(allRepositories.getRepositories()).collect(CodeRepository::getName) :
+                                 toSerialize.toSortedList(new RepositoryComparator(allRepositories.getRepositories()));
+                LOGGER.debug("Serializing modules in order: {}", orderedModules);
+            }
+
+            // Create a class loader that includes the output directory so that previously serialized modules can be loaded
+            try (URLClassLoader outputClassLoader = newClassLoaderWithOutputDirectory(classLoader, outputDirectory))
+            {
+                PureCompilerLoader loader = PureCompilerLoader.newLoader(outputClassLoader);
+                orderedModules.forEach(m -> serializeModules(outputDirectory, classLoader, Sets.immutable.with(m), allRepositories, loader, filePathProvider));
+            }
+            catch (IOException e)
+            {
+                throw new UncheckedIOException("Error closing class loader", e);
+            }
+        }
+        else
+        {
+            serializeModules(outputDirectory, classLoader, toSerialize, allRepositories, PureCompilerLoader.newLoader(classLoader), filePathProvider);
         }
-        CodeRepositorySet resolvedRepositories = builder.build();
-        LOGGER.debug("Resolved repositories: {}", resolvedRepositories.getRepositoryNames());
-        return new RepositoryInfo(resolvedRepositories, toSerialize);
     }
 
-    private static PureRuntime compile(ClassLoader classLoader, CodeRepositorySet codeRepositories)
+    private static void serializeModules(Path outputDirectory, ClassLoader classLoader, SetIterable<String> modulesToSerialize, CodeRepositorySet modulesToCompile, PureCompilerLoader loader, FilePathProvider filePathProvider)
+    {
+        long moduleStart = System.nanoTime();
+        LOGGER.info("Starting compilation and serialization of {}", (modulesToSerialize.size() == 1) ? modulesToSerialize.getAny() : modulesToSerialize);
+        try
+        {
+            LOGGER.debug("Compiling modules {}", modulesToCompile.getRepositoryNames());
+            PureRuntime runtime = compile(classLoader, loader, modulesToCompile);
+            serialize(outputDirectory, modulesToSerialize, runtime, filePathProvider);
+        }
+        finally
+        {
+            long moduleEnd = System.nanoTime();
+            LOGGER.info("Finished compilation and serialization of {} in {}s", (modulesToSerialize.size() == 1) ? modulesToSerialize.getAny() : modulesToSerialize, (moduleEnd - moduleStart) / 1_000_000_000.0);
+        }
+    }
+
+    private static URLClassLoader newClassLoaderWithOutputDirectory(ClassLoader parent, Path outputDirectory)
+    {
+        try
+        {
+            return new URLClassLoader(new URL[]{outputDirectory.toUri().toURL()}, parent);
+        }
+        catch (MalformedURLException e)
+        {
+            throw new RuntimeException("Error creating class loader with output directory: " + outputDirectory, e);
+        }
+    }
+
+    private static PureRuntime compile(ClassLoader classLoader, PureCompilerLoader loader, CodeRepositorySet codeRepositories)
     {
         long start = System.nanoTime();
         LOGGER.info("Starting compilation");
@@ -150,7 +228,6 @@ private static PureRuntime compile(ClassLoader classLoader, CodeRepositorySet co
                     .setTransactionalByDefault(false)
                     .build();
 
-            PureCompilerLoader loader = PureCompilerLoader.newLoader(classLoader);
             MutableList<String> reposToLoad = Lists.mutable.empty();
             MutableList<String> reposToCompile = Lists.mutable.empty();
             codeRepositories.getRepositoryNames().forEach(r -> (loader.canLoad(r) ? reposToLoad : reposToCompile).add(r));
@@ -264,16 +341,4 @@ private static void serialize(Path outputDirectory, SetIterable<String> modules,
             LOGGER.info("Finished serialization in {}s", (end - start) / 1_000_000_000.0);
         }
     }
-
-    private static class RepositoryInfo
-    {
-        private final CodeRepositorySet toCompile;
-        private final SetIterable<String> toSerialize;
-
-        private RepositoryInfo(CodeRepositorySet toCompile, SetIterable<String> toSerialize)
-        {
-            this.toCompile = toCompile;
-            this.toSerialize = toSerialize;
-        }
-    }
 }

legend-pure-core/legend-pure-m3-core/src/main/java/org/finos/legend/pure/m3/serialization/compiler/file/FileDeserializer.java

@@ -1121,7 +1121,7 @@ public Builder withSerializers(ConcreteElementDeserializer elementDeserializer,
         public FileDeserializer build()
         {
             Objects.requireNonNull(this.filePathProvider, "file path provider is required");
-            Objects.requireNonNull(this.elementDeserializer, "concrete element serializer is required");
+            Objects.requireNonNull(this.elementDeserializer, "concrete element deserializer is required");
             Objects.requireNonNull(this.moduleSerializer, "module serializer is required");
             return new FileDeserializer(this.filePathProvider, this.elementDeserializer, this.moduleSerializer);
         }

legend-pure-core/legend-pure-m3-core/src/test/java/org/finos/legend/pure/m3/generator/compiler/TestPureCompilerBinaryGenerator.java

@@ -0,0 +1,186 @@
+// Copyright 2026 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.pure.m3.generator.compiler;
+
+import org.eclipse.collections.api.factory.Lists;
+import org.eclipse.collections.api.factory.Sets;
+import org.eclipse.collections.api.list.MutableList;
+import org.eclipse.collections.api.set.MutableSet;
+import org.finos.legend.pure.m3.serialization.compiler.element.ConcreteElementDeserializer;
+import org.finos.legend.pure.m3.serialization.compiler.file.FileDeserializer;
+import org.finos.legend.pure.m3.serialization.compiler.file.FilePathProvider;
+import org.finos.legend.pure.m3.serialization.compiler.metadata.ModuleManifest;
+import org.finos.legend.pure.m3.serialization.compiler.metadata.ModuleMetadataSerializer;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.stream.Stream;
+
+public class TestPureCompilerBinaryGenerator
+{
+    @ClassRule
+    public static TemporaryFolder TMP = new TemporaryFolder();
+
+    private static final String PLATFORM = "platform";
+    private static final String TEST_REPO = "test_generic_repository";
+    private static final String OTHER_TEST_REPO = "other_test_generic_repository";
+
+    @Test
+    public void testSerializeSingleModule() throws IOException
+    {
+        Path outputDirectory = TMP.newFolder().toPath();
+        PureCompilerBinaryGenerator.serializeModules(outputDirectory, Lists.immutable.with(TEST_REPO));
+
+        FileDeserializer deserializer = newFileDeserializer();
+        assertModuleSerialized(deserializer, outputDirectory, TEST_REPO);
+        assertModuleNotSerialized(deserializer, outputDirectory, PLATFORM);
+        assertModuleNotSerialized(deserializer, outputDirectory, OTHER_TEST_REPO);
+    }
+
+    @Test
+    public void testSerializeAllModules() throws IOException
+    {
+        FileDeserializer deserializer = newFileDeserializer();
+
+        // Check the case where all modules are implicitly specified and serialized individually
+        Path implicitOutputDir = TMP.newFolder().toPath();
+        PureCompilerBinaryGenerator.serializeModules(implicitOutputDir, null, true);
+
+        assertModuleSerialized(deserializer, implicitOutputDir, PLATFORM);
+        assertModuleSerialized(deserializer, implicitOutputDir, TEST_REPO);
+        assertModuleSerialized(deserializer, implicitOutputDir, OTHER_TEST_REPO);
+
+        // Check the case where all modules are explicitly specified and serialized together
+        Path explicitOutputDir = TMP.newFolder().toPath();
+        PureCompilerBinaryGenerator.serializeModules(explicitOutputDir, Lists.immutable.with(PLATFORM, TEST_REPO, OTHER_TEST_REPO), false);
+
+        assertModuleSerialized(deserializer, explicitOutputDir, PLATFORM);
+        assertModuleSerialized(deserializer, explicitOutputDir, TEST_REPO);
+        assertModuleSerialized(deserializer, explicitOutputDir, OTHER_TEST_REPO);
+
+        // Assert that the directories have identical content
+        assertDirectoriesEquivalent(implicitOutputDir, explicitOutputDir);
+    }
+
+    @Test
+    public void testSerializeMultipleButNotAllModules() throws IOException
+    {
+        Path outputDirectory = TMP.newFolder().toPath();
+        PureCompilerBinaryGenerator.serializeModules(outputDirectory, Lists.immutable.with(TEST_REPO, OTHER_TEST_REPO));
+
+        FileDeserializer deserializer = newFileDeserializer();
+        assertModuleSerialized(deserializer, outputDirectory, TEST_REPO);
+        assertModuleSerialized(deserializer, outputDirectory, OTHER_TEST_REPO);
+        assertModuleNotSerialized(deserializer, outputDirectory, PLATFORM);
+    }
+
+    @Test
+    public void testSerializeWithExcludedModule() throws IOException
+    {
+        Path outputDirectory = TMP.newFolder().toPath();
+        PureCompilerBinaryGenerator.serializeModules(outputDirectory, null, null, Lists.immutable.with(OTHER_TEST_REPO));
+
+        FileDeserializer deserializer = newFileDeserializer();
+        assertModuleSerialized(deserializer, outputDirectory, PLATFORM);
+        assertModuleSerialized(deserializer, outputDirectory, TEST_REPO);
+        assertModuleNotSerialized(deserializer, outputDirectory, OTHER_TEST_REPO);
+    }
+
+    private static void assertModuleSerialized(FileDeserializer deserializer, Path outputDirectory, String moduleName)
+    {
+        Assert.assertTrue(moduleName + " manifest should exist", deserializer.moduleManifestExists(outputDirectory, moduleName));
+        ModuleManifest manifest = deserializer.deserializeModuleManifest(outputDirectory, moduleName);
+        Assert.assertEquals(moduleName, manifest.getModuleName());
+        manifest.forEachElement(element ->
+        {
+            String elementPath = element.getPath();
+            Assert.assertTrue(moduleName + " / " + elementPath, deserializer.elementExists(outputDirectory, elementPath));
+        });
+    }
+
+    private static void assertModuleNotSerialized(FileDeserializer deserializer, Path outputDirectory, String moduleName)
+    {
+        Assert.assertFalse(moduleName + " manifest should not exist", deserializer.moduleManifestExists(outputDirectory, moduleName));
+    }
+
+    private static void assertDirectoriesEquivalent(Path dir1, Path dir2)
+    {
+        // First check that relative files paths are the same
+        MutableSet<Path> paths1 = Sets.mutable.empty();
+        MutableSet<Path> paths2 = Sets.mutable.empty();
+        try (Stream<Path> stream1 = Files.walk(dir1);
+             Stream<Path> stream2 = Files.walk(dir2))
+        {
+            stream1.map(dir1::relativize).forEach(paths1::add);
+            stream2.map(dir2::relativize).forEach(paths2::add);
+        }
+        catch (IOException e)
+        {
+            throw new UncheckedIOException(e);
+        }
+        Assert.assertEquals(paths1, paths2);
+
+        MutableList<Path> mismatchFiles = Lists.mutable.empty();
+        paths1.forEach(path ->
+        {
+            Path file1 = dir1.resolve(path);
+            Path file2 = dir2.resolve(path);
+            if (Files.isDirectory(file1))
+            {
+                if (!Files.isDirectory(file2))
+                {
+                    mismatchFiles.add(path);
+                }
+            }
+            else if (Files.isDirectory(file2))
+            {
+                mismatchFiles.add(path);
+            }
+            else
+            {
+                try
+                {
+                    byte[] bytes1 = Files.readAllBytes(file1);
+                    byte[] bytes2 = Files.readAllBytes(file2);
+                    if (!Arrays.equals(bytes1, bytes2))
+                    {
+                        mismatchFiles.add(path);
+                    }
+                }
+                catch (IOException e)
+                {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        });
+        Assert.assertEquals(Lists.fixedSize.empty(), mismatchFiles);
+    }
+
+    private static FileDeserializer newFileDeserializer()
+    {
+        return FileDeserializer.builder()
+                .withFilePathProvider(FilePathProvider.builder().withLoadedExtensions().build())
+                .withConcreteElementDeserializer(ConcreteElementDeserializer.builder().withLoadedExtensions().build())
+                .withModuleMetadataSerializer(ModuleMetadataSerializer.builder().withLoadedExtensions().build())
+                .build();
+    }
+}