Skip to content

Reduce resource pack manifest memory usage #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ABKQPO merged 1 commit into from
Sep 3, 2026
Merged

Reduce resource pack manifest memory usage #68

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 9 additions & 27 deletions src/main/java/com/hfstudio/guidenh/guide/internal/datadriven/DataDrivenGuideLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,33 +89,13 @@ private record NamespaceRoot(String namespace, File directory, boolean allowDire

private static volatile @Nullable ScanCache lastScanCache = null;

private record ScanCache(List<File> packRoots, String folder,
List<ResourcePackContentManifest.Pack> contentManifest, ScanResult result,
private record ScanCache(String folder, List<ResourcePackContentManifest.Pack> contentManifest, ScanResult result,
Map<ResourceLocation, List<PackCandidate>> pagePackIndexSnapshot,
Map<ResourceLocation, List<PackCandidate>> assetPackIndexSnapshot,
Map<ResourcePackViewKey, List<String>> langFilePathsSnapshot) {

boolean matchesPackMetadata(List<File> roots, String f) {
if (!folder.equals(f) || !packRoots.equals(roots) || roots.size() != contentManifest.size()) {
return false;
}
for (int i = 0; i < roots.size(); i++) {
File current = roots.get(i);
File previous = packRoots.get(i);
// ZIP contents can only change when the archive's size or timestamp changes.
// Directory packs are deliberately rescanned so file-level hot reloads remain
// visible even when the directory timestamp is unchanged on some file systems.
if (current.isDirectory() || previous.isDirectory()
|| current.length() != previous.length()
|| current.lastModified() != previous.lastModified()) {
return false;
}
}
return true;
}

boolean matches(List<File> roots, String f, List<ResourcePackContentManifest.Pack> manifest) {
return folder.equals(f) && packRoots.equals(roots) && contentManifest.equals(manifest);
boolean matches(String f, List<ResourcePackContentManifest.Pack> manifest) {
return folder.equals(f) && contentManifest.equals(manifest);
}
}

Expand All @@ -132,9 +112,12 @@ public static ScanResult scanAndBuildAll(String folder, Iterable<? extends IReso
var resolvedPacks = toList(activeResourcePacks);
var packRoots = resolvePackRoots(resolvedPacks);
ScanCache cache = lastScanCache;
var contentManifest = cache != null && cache.matchesPackMetadata(packRoots, folder) ? cache.contentManifest()
: ResourcePackContentManifest.capture(packRoots, folder);
if (cache != null && cache.matches(packRoots, folder, contentManifest)) {
var contentManifest = ResourcePackContentManifest.capture(
packRoots,
folder,
cache != null && cache.folder()
.equals(folder) ? cache.contentManifest() : List.of());
if (cache != null && cache.matches(folder, contentManifest)) {
indexReady = true;
pagePackIndex.putAll(cache.pagePackIndexSnapshot());
assetPackIndex.putAll(cache.assetPackIndexSnapshot());
Expand Down Expand Up @@ -176,7 +159,6 @@ public static ScanResult scanAndBuildAll(String folder, Iterable<? extends IReso
// during the snapshot (Map.copyOf on ConcurrentHashMap can throw on concurrent read).
if (!resolvedPacks.isEmpty() && !guides.isEmpty()) {
lastScanCache = new ScanCache(
List.copyOf(packRoots),
folder,
contentManifest,
new ScanResult(guides, pagePaths, freezeDiscoveredLanguages(discoveredLanguages)),
Expand Down
175 changes: 110 additions & 65 deletions ...main/java/com/hfstudio/guidenh/guide/internal/datadriven/ResourcePackContentManifest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,50 @@
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.zip.CRC32;
import java.util.zip.ZipFile;

/**
* Lightweight content inventory used to decide whether a resource-pack scan can be reused.
* The inventory is only a cache key. GuideNH entries in directory packs include a streamed
* content checksum so a same-size edit (or an edit with a preserved timestamp) still invalidates
* the scan cache. Other assets keep metadata only because their loading is owned by Minecraft's
* resource manager, while no per-file objects are retained after the scan.
* The inventory is only a cache key. Directory packs include a streamed checksum for GuideNH
* files and language files, so a same-size edit (or an edit with a preserved timestamp) still
* invalidates the scan cache. ZIP packs need only immutable archive metadata: their internal
* entries are never retained or enumerated for this purpose.
*/
public class ResourcePackContentManifest {

/**
* Compact cache key for a resource pack. The scanner still visits every file, but the
* resulting cache retains only a count and an order-independent fingerprint instead of one
* object (and path string) per file.
* Compact cache key for a resource pack. Directory packs retain only an order-independent
* fingerprint. ZIP packs retain only their normalized path and archive metadata.
*/
public record Pack(File root, boolean directory, int entryCount, long fingerprint) {}
public record Pack(String path, boolean directory, long size, long lastModified, int entryCount, long fingerprint) {

public boolean matchesZip(File root) {
return !directory && !root.isDirectory()
&& path.equals(normalizePath(root))
&& size == root.length()
&& lastModified == root.lastModified();
}
}

public static List<Pack> capture(List<File> roots, String guideFolder) {
return capture(roots, guideFolder, List.of());
}

/**
* Reuses the immutable ZIP metadata from the last scan. Directory packs remain content-aware
* and are rescanned on every reload because changing a child does not reliably update the
* directory timestamp across filesystems.
*/
public static List<Pack> capture(List<File> roots, String guideFolder, List<Pack> previous) {
var result = new ArrayList<Pack>(roots.size());
for (File root : roots) {
result.add(root.isDirectory() ? captureDirectory(root, guideFolder) : captureZip(root));
for (int index = 0; index < roots.size(); index++) {
File root = roots.get(index);
Pack cached = index < previous.size() ? previous.get(index) : null;
if (!root.isDirectory() && cached != null && cached.matchesZip(root)) {
result.add(cached);
} else {
result.add(root.isDirectory() ? captureDirectory(root, guideFolder) : captureZip(root));
}
}
return List.copyOf(result);
}
Expand All @@ -41,47 +61,61 @@ public static Pack captureDirectory(File root, String guideFolder) {
.normalize();
var hashes = new LongHashBuffer();

collectFiles(absoluteRoot.resolve("assets"), absoluteRoot, hashes);
Path assets = absoluteRoot.resolve("assets");
collectAssets(assets, absoluteRoot, guideFolder, hashes);
try (var children = Files.list(absoluteRoot)) {
children.filter(Files::isDirectory)
.filter(
path -> !"assets".equals(
path.getFileName()
.toString()))
.forEach(path -> collectFiles(path.resolve(guideFolder), absoluteRoot, hashes));
.forEach(path -> collectNativeNamespace(path, absoluteRoot, guideFolder, hashes));
} catch (IOException e) {
hashes.add(entryHash("<unreadable-root>", -1L, -1L, -1L));
}

return new Pack(root, true, hashes.size(), hashes.fingerprint());
return new Pack(
normalizePath(root),
true,
root.length(),
root.lastModified(),
hashes.size(),
hashes.fingerprint());
}

public static void collectFiles(Path directory, Path root, LongHashBuffer hashes) {
private static void collectAssets(Path assets, Path root, String guideFolder, LongHashBuffer hashes) {
if (!Files.isDirectory(assets)) return;
try (var namespaces = Files.list(assets)) {
namespaces.filter(Files::isDirectory)
.forEach(namespace -> {
collectFiles(namespace.resolve(guideFolder), root, hashes);
collectLangFiles(namespace.resolve("lang"), root, hashes);
});
} catch (IOException e) {
hashes.add(entryHash("assets/<unreadable>", -1L, -1L, -1L));
}
}

private static void collectNativeNamespace(Path namespace, Path root, String guideFolder, LongHashBuffer hashes) {
collectFiles(namespace.resolve(guideFolder), root, hashes);
if (guideFolder.equals(
namespace.getFileName()
.toString())) {
collectFiles(namespace, root, hashes);
}
collectLangFiles(namespace.resolve("lang"), root, hashes);
}

private static void collectLangFiles(Path directory, Path root, LongHashBuffer hashes) {
if (!Files.isDirectory(directory)) return;
byte[] contentBuffer = new byte[16 * 1024];
try (var files = Files.walk(directory)) {
files.filter(Files::isRegularFile)
.forEach(path -> {
try {
var attributes = Files.readAttributes(path, BasicFileAttributes.class);
String relativePath = root.relativize(path)
.toString()
.replace(File.separatorChar, '/');
long contentHash = shouldHashContent(relativePath) ? contentChecksum(path, contentBuffer) : -1L;
hashes.add(
entryHash(
relativePath,
attributes.size(),
attributes.lastModifiedTime()
.toMillis(),
contentHash));
} catch (IOException e) {
String relativePath = root.relativize(path)
.toString()
.replace(File.separatorChar, '/');
hashes.add(entryHash(relativePath, -1L, -1L, -1L));
}
});
.filter(
path -> path.getFileName()
.toString()
.endsWith(".lang"))
.forEach(path -> addFile(path, root, hashes, contentBuffer));
} catch (IOException e) {
hashes.add(
entryHash(
Expand All @@ -94,21 +128,26 @@ public static void collectFiles(Path directory, Path root, LongHashBuffer hashes
}
}

public static Pack captureZip(File root) {
var hashes = new LongHashBuffer();
try (var zip = new ZipFile(root)) {
var zipEntries = zip.entries();
while (zipEntries.hasMoreElements()) {
var entry = zipEntries.nextElement();
if (!entry.isDirectory() && entry.getName()
.startsWith("assets/")) {
hashes.add(entryHash(entry.getName(), entry.getSize(), entry.getTime(), entry.getCrc()));
}
}
public static void collectFiles(Path directory, Path root, LongHashBuffer hashes) {
if (!Files.isDirectory(directory)) return;
byte[] contentBuffer = new byte[16 * 1024];
try (var files = Files.walk(directory)) {
files.filter(Files::isRegularFile)
.forEach(path -> addFile(path, root, hashes, contentBuffer));
} catch (IOException e) {
hashes.add(entryHash("<unreadable-zip>", -1L, -1L, -1L));
hashes.add(
entryHash(
root.relativize(directory)
.toString()
.replace(File.separatorChar, '/') + "/<unreadable>",
-1L,
-1L,
-1L));
}
return new Pack(root, false, hashes.size(), hashes.fingerprint());
}

public static Pack captureZip(File root) {
return new Pack(normalizePath(root), false, root.length(), root.lastModified(), 0, 0L);
}

public static long entryHash(String path, long size, long modified, long crc) {
Expand Down Expand Up @@ -147,23 +186,29 @@ private static long contentChecksum(Path path, byte[] buffer) {
}
}

private static boolean isGuideRelevantPath(String path) {
return path.endsWith(".lang") || path.contains("/guidenh/") || path.endsWith("/guidenh");
private static void addFile(Path path, Path root, LongHashBuffer hashes, byte[] contentBuffer) {
String relativePath = root.relativize(path)
.toString()
.replace(File.separatorChar, '/');
try {
var attributes = Files.readAttributes(path, BasicFileAttributes.class);
hashes.add(
entryHash(
relativePath,
attributes.size(),
attributes.lastModifiedTime()
.toMillis(),
contentChecksum(path, contentBuffer)));
} catch (IOException e) {
hashes.add(entryHash(relativePath, -1L, -1L, -1L));
}
}

private static boolean shouldHashContent(String path) {
if (isGuideRelevantPath(path)) {
return true;
}
String lower = path.toLowerCase(Locale.ROOT);
return lower.endsWith(".png") || lower.endsWith(".jpg")
|| lower.endsWith(".jpeg")
|| lower.endsWith(".gif")
|| lower.endsWith(".webp")
|| lower.endsWith(".mcmeta")
|| lower.endsWith(".json")
|| lower.endsWith(".snbt")
|| lower.endsWith(".nbt");
private static String normalizePath(File root) {
return root.toPath()
.toAbsolutePath()
.normalize()
.toString();
}

/** Small primitive buffer used only during a scan; no per-entry objects survive the scan. */
Expand Down