Skip to content

Commit 710551f

Browse files
SkidamekSkidamek
committed
Improve isUpdate performance
1 parent a74bee9 commit 710551f

2 files changed

Lines changed: 95 additions & 27 deletions

File tree

‎core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,14 @@ private FileMetadataCache(Path dbPath) {
6161
}
6262

6363
public String getOrComputeHash(Path file) throws IOException {
64+
BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class);
65+
return getOrComputeHashWithAttributes(file, attrs);
66+
}
67+
68+
public String getOrComputeHashWithAttributes(Path file, BasicFileAttributes attrs) {
6469
Path absPath = file.toAbsolutePath().normalize();
6570
String pathKey = absPath.toString();
6671

67-
BasicFileAttributes attrs = Files.readAttributes(absPath, BasicFileAttributes.class);
6872
long currentSize = attrs.size();
6973
long currentTime = attrs.lastModifiedTime().toMillis();
7074
String currentFileKey = attrs.fileKey() != null ? attrs.fileKey().toString() : "null";
@@ -107,6 +111,15 @@ public String getHashOrNull(Path path) {
107111
}
108112
}
109113

114+
public String getHashOrNullWithAttributes(Path path, BasicFileAttributes attrs) {
115+
try {
116+
return getOrComputeHashWithAttributes(path, attrs);
117+
} catch (Exception e) {
118+
LOGGER.error("Failed to compute hash for path: {}", path, e);
119+
return null;
120+
}
121+
}
122+
110123
public boolean fastHashCompare(Path file1, Path file2) throws IOException {
111124
if (!Files.exists(file1) || !Files.exists(file2)) return false;
112125

‎loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java‎

Lines changed: 81 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package pl.skidam.automodpack_loader_core.client;
22

3+
import org.jetbrains.annotations.NotNull;
34
import pl.skidam.automodpack_core.auth.Secrets;
45
import pl.skidam.automodpack_core.config.ConfigTools;
56
import pl.skidam.automodpack_core.config.Jsons;
@@ -16,6 +17,7 @@
1617
import java.io.*;
1718
import java.net.*;
1819
import java.nio.file.*;
20+
import java.nio.file.attribute.BasicFileAttributes;
1921
import java.security.cert.CertificateEncodingException;
2022
import java.security.cert.X509Certificate;
2123
import java.util.*;
@@ -50,40 +52,93 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac
5052
return new UpdateCheckResult(true, serverModpackContent.list);
5153
}
5254

53-
LOGGER.info("Indexing file system...");
54-
var start = System.currentTimeMillis();
55-
56-
Set<Path> existingFileTree;
57-
try (var stream = Files.walk(modpackDir)) {
58-
existingFileTree = stream.collect(Collectors.toSet());
59-
} catch (IOException e) {
60-
LOGGER.error("Failed to walk directory", e);
61-
return new UpdateCheckResult(true, serverModpackContent.list);
62-
}
63-
6455
LOGGER.info("Verifying content against server list...");
56+
var start = System.currentTimeMillis();
6557

6658
Set<Jsons.ModpackContentFields.ModpackContentItem> filesToUpdate = ConcurrentHashMap.newKeySet();
6759

60+
// Group & Sort Server Files (Optimizes Disk Seek Pattern)
61+
// Grouping by parent folder ensures we process the disk sequentially (Dir A, then Dir B).
62+
// TreeMap ensures alphabetical order of directories (HDD friendly).
63+
Map<Path, List<Jsons.ModpackContentFields.ModpackContentItem>> itemsByDir =
64+
serverModpackContent.list.stream()
65+
.collect(Collectors.groupingBy(
66+
item -> SmartFileUtils.getPath(modpackDir, item.file).getParent(),
67+
TreeMap::new,
68+
Collectors.toList()
69+
));
70+
6871
try (var cache = FileMetadataCache.open(hashCacheDBFile)) {
69-
serverModpackContent.list.forEach(serverItem -> {
70-
Path serverItemPath = SmartFileUtils.getPath(modpackDir, serverItem.file);
71-
if (!existingFileTree.contains(serverItemPath)) {
72-
filesToUpdate.add(serverItem); // File is missing
73-
return;
74-
} else if (serverItem.editable) { // TODO check if this is enough of a check, what if user already had a file but there's provided the same by a new modpack version which wasnt in the modpack before?
75-
LOGGER.debug("Skipping editable file hash check: {}", serverItem.file);
76-
return;
72+
73+
// Process Directory by Directory
74+
for (Map.Entry<Path, List<Jsons.ModpackContentFields.ModpackContentItem>> entry : itemsByDir.entrySet()) {
75+
Path parentDir = entry.getKey();
76+
List<Jsons.ModpackContentFields.ModpackContentItem> itemsInDir = entry.getValue();
77+
78+
// If directory is missing, all items in it are missing.
79+
if (!Files.exists(parentDir)) {
80+
filesToUpdate.addAll(itemsInDir);
81+
continue;
7782
}
7883

79-
String hash = cache.getHashOrNull(serverItemPath);
80-
if (hash != null && hash.equals(serverItem.sha1)) {
81-
return; // File is up to date
84+
// Read all file attributes in this folder in ONE pass.
85+
// This map will hold "FileName" -> "Attributes"
86+
Map<String, BasicFileAttributes> diskFiles = new HashMap<>();
87+
88+
try {
89+
// walkFileTree with depth 1 is efficient on Windows (gets attributes for free within a single syscall)
90+
Files.walkFileTree(parentDir, EnumSet.noneOf(FileVisitOption.class), 1, new SimpleFileVisitor<>() {
91+
@NotNull @Override
92+
public FileVisitResult visitFile(@NotNull Path file, @NotNull BasicFileAttributes attrs) {
93+
diskFiles.put(file.getFileName().toString(), attrs);
94+
return FileVisitResult.CONTINUE;
95+
}
96+
97+
@NotNull @Override
98+
public FileVisitResult visitFileFailed(@NotNull Path file, @NotNull IOException exc) {
99+
return FileVisitResult.CONTINUE; // Handle locked files or permission errors gracefully
100+
}
101+
});
102+
} catch (IOException e) {
103+
LOGGER.warn("Failed to inspect directory: {}", parentDir, e);
104+
filesToUpdate.addAll(itemsInDir);
105+
continue;
82106
}
83107

84-
// This file needs to be updated
85-
filesToUpdate.add(serverItem);
86-
});
108+
// Check Individual Files in a given directory (Pure RAM logic, 0 IO)
109+
for (var serverItem : itemsInDir) {
110+
String fileName = Paths.get(serverItem.file).getFileName().toString();
111+
BasicFileAttributes diskAttrs = diskFiles.get(fileName);
112+
113+
if (diskAttrs == null) {
114+
// File does not exist in the directory map
115+
filesToUpdate.add(serverItem);
116+
} else {
117+
if (serverItem.editable) { // TODO check if this is enough of a check, what if user already had a file but there's provided the same by a new modpack version which wasn't in the modpack before?
118+
LOGGER.debug("Skipping editable file hash check: {}", serverItem.file);
119+
continue;
120+
}
121+
122+
// Check Size first from already read attributes
123+
if (diskAttrs.size() != Long.parseLong(serverItem.size)) {
124+
filesToUpdate.add(serverItem);
125+
continue;
126+
}
127+
128+
// Finally, check Hash
129+
// We pass 'diskAttrs' to the cache so it doesn't need to re-stat the file.
130+
String hash = cache.getHashOrNullWithAttributes(parentDir.resolve(fileName), diskAttrs);
131+
132+
if (!serverItem.sha1.equalsIgnoreCase(hash)) {
133+
filesToUpdate.add(serverItem);
134+
}
135+
}
136+
}
137+
}
138+
} catch (Exception e) {
139+
LOGGER.error("Error during update check", e);
140+
// Fail-safe: assume update needed if process crashes
141+
return new UpdateCheckResult(true, serverModpackContent.list);
87142
}
88143

89144
if (!filesToUpdate.isEmpty()) {
@@ -99,7 +154,7 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac
99154

100155
for (Jsons.ModpackContentFields.ModpackContentItem clientItem : clientModpackContent.list) {
101156
if (!serverFileSet.contains(clientItem.file)) {
102-
LOGGER.info("Found file marked for deletion (its no longer on server): {}", clientItem.file);
157+
LOGGER.info("Found file marked for deletion: {}", clientItem.file);
103158
return new UpdateCheckResult(true, Set.of());
104159
}
105160
}

0 commit comments

Comments
 (0)