Implement generation of split-source Manifest entries (#50, updated) (#66)

lukebemish · shartte · web-flow · commit f770c88fac93 · 2025-06-28T00:23:34.000+02:00
This is a rebased version of #50 (made as a separate PR because I lack appropriate permissions to do otherwise); from a quick test, its behaviour seems to match that of neoforged/NeoGradle#257, putting the generated manifest info in the MANIFEST.MF of the client-extra jar. --------- Co-authored-by: Sebastian Hartte <shartte@users.noreply.github.com>
diff --git a/build.gradle b/build.gradle
@@ -163,6 +163,11 @@ idea {
                     programParameters = "--help"
                     moduleRef(project, sourceSets.main)
                 }
+                "Run Neoforge 1.21.6 (joined) + Parchment"(Application) {
+                    mainClass = mainClassName
+                    programParameters = "run --dist joined --neoforge net.neoforged:neoforge:21.6.1-beta:userdev --add-repository=https://maven.parchmentmc.org --parchment-data=org.parchmentmc.data:parchment-1.21.5:2025.06.15@zip --parchment-conflict-prefix=p_ --write-result=compiled:build/minecraft-1.21.6.jar --write-result=clientResources:build/client-extra-1.21.6.jar --write-result=sources:build/minecraft-sources-1.21.6.jar"
+                    moduleRef(project, sourceSets.main)
+                }
                 "Run Neoforge 1.21 (joined) + Parchment"(Application) {
                     mainClass = mainClassName
                     programParameters = "run --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --add-repository=https://maven.parchmentmc.org --parchment-data=org.parchmentmc.data:parchment-1.21:2024.06.23@zip --parchment-conflict-prefix=p_ --write-result=compiled:build/minecraft-1.21.jar --write-result=clientResources:build/client-extra-1.21.jar --write-result=sources:build/minecraft-sources-1.21.jar"
diff --git a/src/main/java/net/neoforged/neoform/runtime/actions/SplitResourcesFromClassesAction.java b/src/main/java/net/neoforged/neoform/runtime/actions/SplitResourcesFromClassesAction.java
@@ -2,32 +2,74 @@
 
 import net.neoforged.neoform.runtime.cache.CacheKeyBuilder;
 import net.neoforged.neoform.runtime.engine.ProcessingEnvironment;
+import net.neoforged.srgutils.IMappingFile;
+import org.jetbrains.annotations.Nullable;
 
-import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 import java.util.function.Predicate;
-import java.util.jar.JarEntry;
-import java.util.jar.JarInputStream;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
 import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 /**
  * Copies a Jar file while applying a filename filter.
+ * <p>Optionally, this also {@link #generateSplitManifest creates and injects} a {@code MANIFEST.MF} file that details files that are exclusive
+ * to the distribution of Minecraft being processed by this action.
  */
 public final class SplitResourcesFromClassesAction extends BuiltInAction {
+
+    /**
+     * @see #generateSplitManifest
+     */
+    public static final String INPUT_OTHER_DIST_JAR = "otherDistJar";
+    /**
+     * @see #generateSplitManifest
+     */
+    public static final String INPUT_MAPPINGS = "mappings";
+
+    /**
+     * Use a fixed timestamp for the manifest entry.
+     */
+    private static final LocalDateTime MANIFEST_TIME = LocalDateTime.of(2000, 1, 1, 0, 0, 0, 0);
+
     /**
      * Patterns for filenames that should not be written to either output jar.
      */
     private final List<Pattern> denyListPatterns = new ArrayList<>();
 
+    /**
+     * When non-null, the action expects additional inputs ({@link #INPUT_OTHER_DIST_JAR} and {@link #INPUT_MAPPINGS})
+     * pointing to the Jar file of the *other* distribution (i.e. this action processes the client resources,
+     * then the other distribution jar is the server jar).
+     * The mapping file is required to produce a Manifest using named file names instead of obfuscated names.
+     */
+    @Nullable
+    private GenerateDistManifestSettings generateDistManifestSettings;
+
     @Override
     public void run(ProcessingEnvironment environment) throws IOException, InterruptedException {
         var inputJar = environment.getRequiredInputPath("input");
+        Path otherDistJarPath = null;
+        Path mappingsPath = null;
+        if (generateDistManifestSettings != null) {
+            otherDistJarPath = environment.getRequiredInputPath(INPUT_OTHER_DIST_JAR);
+            mappingsPath = environment.getRequiredInputPath(INPUT_MAPPINGS);
+        }
+
         var classesJar = environment.getOutputPath("output");
         var resourcesJar = environment.getOutputPath("resourcesOutput");
 
@@ -39,19 +81,35 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
                     .asMatchPredicate();
         }
 
-        try (var is = new JarInputStream(new BufferedInputStream(Files.newInputStream(inputJar)));
+        try (var jar = new ZipFile(inputJar.toFile());
              var classesFileOut = new BufferedOutputStream(Files.newOutputStream(classesJar));
              var resourcesFileOut = new BufferedOutputStream(Files.newOutputStream(resourcesJar));
              var classesJarOut = new JarOutputStream(classesFileOut);
              var resourcesJarOut = new JarOutputStream(resourcesFileOut);
         ) {
-            // Ignore any entry that's not allowed
-            JarEntry entry;
-            while ((entry = is.getNextJarEntry()) != null) {
+            if (generateDistManifestSettings != null) {
+                generateDistSourceManifest(
+                        generateDistManifestSettings.distId(),
+                        jar,
+                        generateDistManifestSettings.otherDistId(),
+                        otherDistJarPath,
+                        mappingsPath,
+                        resourcesJarOut
+                );
+            }
+
+            var entries = jar.entries();
+            while (entries.hasMoreElements()) {
+                var entry = entries.nextElement();
                 if (entry.isDirectory()) {
                     continue; // For simplicity, we ignore directories completely
                 }
 
+                // If this task generates its own manifest, ignore any manifests found in the input jar
+                if (generateDistManifestSettings != null && entry.getName().equals(JarFile.MANIFEST_NAME)) {
+                    continue;
+                }
+
                 var filename = entry.getName();
 
                 // Skip anything that looks like a signature file
@@ -62,12 +120,77 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
                 var destinationStream = filename.endsWith(".class") ? classesJarOut : resourcesJarOut;
 
                 destinationStream.putNextEntry(entry);
-                is.transferTo(destinationStream);
+                try (var is = jar.getInputStream(entry)) {
+                    is.transferTo(destinationStream);
+                }
                 destinationStream.closeEntry();
             }
         }
     }
 
+    private static void generateDistSourceManifest(String distId,
+                                                   ZipFile jar,
+                                                   String otherDistId,
+                                                   Path otherDistJarPath,
+                                                   Path mappingsPath,
+                                                   JarOutputStream resourcesJarOut) throws IOException {
+        var mappings = mappingsPath != null ? IMappingFile.load(mappingsPath.toFile()) : null;
+
+        // Use the time-stamp of either of the two input files (whichever is newer)
+        var ourFiles = getFileIndex(jar);
+        ourFiles.remove(JarFile.MANIFEST_NAME);
+        Set<String> theirFiles;
+        try (var otherDistJar = new ZipFile(otherDistJarPath.toFile())) {
+            theirFiles = getFileIndex(otherDistJar);
+        }
+        theirFiles.remove(JarFile.MANIFEST_NAME);
+
+        var manifest = new Manifest();
+        manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+        manifest.getMainAttributes().putValue("Minecraft-Dists", distId + " " + otherDistId);
+
+        addSourceDistEntries(ourFiles, theirFiles, distId, mappings, manifest);
+        addSourceDistEntries(theirFiles, ourFiles, otherDistId, mappings, manifest);
+
+        var manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
+        manifestEntry.setTimeLocal(MANIFEST_TIME);
+        resourcesJarOut.putNextEntry(manifestEntry);
+        manifest.write(resourcesJarOut);
+        resourcesJarOut.closeEntry();
+    }
+
+    private static void addSourceDistEntries(Set<String> distFiles,
+                                             Set<String> otherDistFiles,
+                                             String dist,
+                                             IMappingFile mappings,
+                                             Manifest manifest) {
+        for (var file : distFiles) {
+            if (!otherDistFiles.contains(file)) {
+                var fileAttr = new Attributes(1);
+                fileAttr.putValue("Minecraft-Dist", dist);
+
+                if (mappings != null && file.endsWith(".class")) {
+                    file = mappings.remapClass(file.substring(0, file.length() - ".class".length())) + ".class";
+                }
+                manifest.getEntries().put(file, fileAttr);
+            }
+        }
+    }
+
+    private static Set<String> getFileIndex(ZipFile zipFile) {
+        var result = new HashSet<String>(zipFile.size());
+
+        var entries = zipFile.entries();
+        while (entries.hasMoreElements()) {
+            ZipEntry entry = entries.nextElement();
+            if (!entry.isDirectory()) {
+                result.add(entry.getName());
+            }
+        }
+
+        return result;
+    }
+
     /**
      * Adds a regular expression for filenames that should be filtered out completely.
      */
@@ -77,9 +200,38 @@ public void addDenyPatterns(String... patterns) {
         }
     }
 
+    /**
+     * Enable generation of a Jar manifest in the output resources jar which contains
+     * entries detailing which distribution each file came from.
+     * <p>This adds required inputs {@link #INPUT_MAPPINGS} and {@link #INPUT_OTHER_DIST_JAR} to this action.
+     * <p>Common values for distributions are {@code client} and {@code server}.
+     *
+     * @param distId      The name for the distribution that the main input file is from. It is used in the
+     *                    generated manifest for files that are only present in the main input, but not in the
+     *                    {@linkplain #INPUT_OTHER_DIST_JAR jar file of the other distribution}.
+     * @param otherDistId The name for the Minecraft distribution for the jar file given in {@link #INPUT_OTHER_DIST_JAR}.
+     *                    It is used in the generated manifest for files that are only present in that jar file.
+     */
+    public void generateSplitManifest(String distId, String otherDistId) {
+        generateDistManifestSettings = new GenerateDistManifestSettings(
+                Objects.requireNonNull(distId, "distId"),
+                Objects.requireNonNull(otherDistId, "otherDistId")
+        );
+    }
+
     @Override
     public void computeCacheKey(CacheKeyBuilder ck) {
         super.computeCacheKey(ck);
         ck.addStrings("deny patterns", denyListPatterns.stream().map(Pattern::pattern).toList());
+        if (generateDistManifestSettings != null) {
+            ck.add("generate dist manifest - our dist", generateDistManifestSettings.distId);
+            ck.add("generate dist manifest - other dist", generateDistManifestSettings.otherDistId);
+        }
+    }
+
+    private record GenerateDistManifestSettings(
+            String distId,
+            String otherDistId
+    ) {
     }
 }
diff --git a/src/main/java/net/neoforged/neoform/runtime/config/neoform/NeoFormDistConfig.java b/src/main/java/net/neoforged/neoform/runtime/config/neoform/NeoFormDistConfig.java
@@ -21,6 +21,10 @@ public NeoFormDistConfig(NeoFormConfig config, String dist) {
         this.dist = dist;
     }
 
+    public String dist() {
+        return dist;
+    }
+
     public int javaVersion() {
         return config.javaVersion();
     }
diff --git a/src/main/java/net/neoforged/neoform/runtime/engine/NeoFormEngine.java b/src/main/java/net/neoforged/neoform/runtime/engine/NeoFormEngine.java
@@ -324,6 +324,26 @@ private void addNodeForStep(ExecutionGraph graph, NeoFormDistConfig config, NeoF
                 var action = new SplitResourcesFromClassesAction();
                 // The Minecraft jar contains nothing of interest in META-INF, and the signature files are useless.
                 action.addDenyPatterns("META-INF/.*");
+
+                // When generating Minecraft artifacts that join the client and server, we generate a MANIFEST.MF that
+                // indicates files exclusive to one or the other. This started in Minecraft 1.21.6.
+                if (processGeneration.generateDistSourceManifest() && config.dist().equals("joined")) {
+                    if ("stripClient".equals(step.getId())) {
+                        // Prefer the already extracted server, otherwise download it
+                        var serverJarInput = graph.hasOutput("extractServer", "output") ?
+                                graph.getRequiredOutput("extractServer", "output").asInput()
+                                : graph.getRequiredOutput("downloadServer", "output").asInput();
+
+                        action.generateSplitManifest("client", "server");
+                        builder.input(SplitResourcesFromClassesAction.INPUT_OTHER_DIST_JAR, serverJarInput);
+                        builder.input(SplitResourcesFromClassesAction.INPUT_MAPPINGS, graph.getRequiredOutput("mergeMappings", "output").asInput());
+                    } else if ("stripServer".equals(step.getId())) {
+                        action.generateSplitManifest("server", "client");
+                        builder.input(SplitResourcesFromClassesAction.INPUT_OTHER_DIST_JAR, graph.getRequiredOutput("downloadClient", "output").asInput());
+                        builder.input(SplitResourcesFromClassesAction.INPUT_MAPPINGS, graph.getRequiredOutput("mergeMappings", "output").asInput());
+                    }
+                }
+
                 processGeneration.getAdditionalDenyListForMinecraftJars().forEach(action::addDenyPatterns);
                 builder.action(action);
             }
@@ -401,6 +421,7 @@ private void applyFunctionToNode(NeoFormStep step, NeoFormFunction function, Exe
                 if ("output".equals(variable)) {
                     var type = switch (step.type()) {
                         case "mergeMappings" -> NodeOutputType.TSRG;
+                        case "generateSplitManifest" -> NodeOutputType.JAR_MANIFEST;
                         default -> NodeOutputType.JAR;
                     };
                     if (!builder.hasOutput(variable)) {
diff --git a/src/main/java/net/neoforged/neoform/runtime/engine/ProcessGeneration.java b/src/main/java/net/neoforged/neoform/runtime/engine/ProcessGeneration.java
@@ -43,6 +43,7 @@ public int compareTo(@NotNull ProcessGeneration.MinecraftReleaseVersion o) {
 
     private static final MinecraftReleaseVersion MC_1_17_1 = new MinecraftReleaseVersion(1, 17, 1);
     private static final MinecraftReleaseVersion MC_1_20_1 = new MinecraftReleaseVersion(1, 20, 1);
+    private static final MinecraftReleaseVersion MC_1_21_6 = new MinecraftReleaseVersion(1, 21, 6);
 
     /**
      * Indicates whether the Minecraft server jar file contains third party
@@ -64,6 +65,12 @@ public int compareTo(@NotNull ProcessGeneration.MinecraftReleaseVersion o) {
      */
     private boolean supportsSideAnnotationStripping;
 
+    /**
+     * Enables generation of the MANIFEST.MF in the client and server resource files that
+     * indicates which distribution each file came from. Only applies to joined distributions.
+     */
+    private boolean generateDistSourceManifest;
+
     /**
      * For (Neo)Forge 1.20.1 and below, we have to remap method and field names from
      * SRG to official names for development.
@@ -94,6 +101,9 @@ static ProcessGeneration fromMinecraftVersion(String minecraftVersion) {
         // In 1.20.2 and later, NeoForge switched to Mojmap at runtime and sources defined in Mojmap
         result.sourcesUseIntermediaryNames = isLessThanOrEqualTo(releaseVersion, MC_1_20_1);
 
+        // In 1.21.6 and later, manifest entries should be generated as they may be used instead of RuntimeDistCleaner
+        result.generateDistSourceManifest = isGreaterThanOrEqualTo(releaseVersion, MC_1_21_6);
+
         result.supportsSideAnnotationStripping = isLessThanOrEqualTo(releaseVersion, MC_1_20_1);
 
         return result;
@@ -106,6 +116,13 @@ private static boolean isLessThanOrEqualTo(@Nullable MinecraftReleaseVersion rel
         return releaseVersion.compareTo(version) <= 0;
     }
 
+    private static boolean isGreaterThanOrEqualTo(@Nullable MinecraftReleaseVersion releaseVersion, MinecraftReleaseVersion version) {
+        if (releaseVersion == null) {
+            return true; // We're working with a snapshot version, which we always use the latest processes for
+        }
+        return releaseVersion.compareTo(version) >= 0;
+    }
+
     /**
      * Does the Minecraft source code that MCP/NeoForm creates use SRG names?
      */
@@ -120,6 +137,14 @@ public boolean supportsSideAnnotationStripping() {
         return supportsSideAnnotationStripping;
     }
 
+    /**
+     * Does the FML version on that MC generation support use of MANIFEST.MF entries
+     * for filtering out dist-specific classes in dev? (When using the joined distribution)
+     */
+    public boolean generateDistSourceManifest() {
+        return generateDistSourceManifest;
+    }
+
     /**
      * Allows additional resources to be completely removed from Minecraft jars before processing them.
      */
diff --git a/src/main/java/net/neoforged/neoform/runtime/graph/NodeOutputType.java b/src/main/java/net/neoforged/neoform/runtime/graph/NodeOutputType.java
@@ -2,6 +2,7 @@
 
 public enum NodeOutputType {
     JAR(".jar"),
+    JAR_MANIFEST(".MF"),
     TXT(".txt"),
     ZIP(".zip"),
     JSON(".json"),

Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,10 @@ public NeoFormDistConfig(NeoFormConfig config, String dist) {`
`21`	`21`	`this.dist = dist;`
`22`	`22`	`}`
`23`	`23`
	`24`	`+ public String dist() {`
	`25`	`+ return dist;`
	`26`	`+ }`
	`27`	`+`
`24`	`28`	`public int javaVersion() {`
`25`	`29`	`return config.javaVersion();`
`26`	`30`	`}`