From 9b0a378c1d7d812444c4b9a851572b4eb8e85113 Mon Sep 17 00:00:00 2001 From: skidam Date: Sat, 24 Jan 2026 20:05:11 +0100 Subject: [PATCH 01/22] Refactor global variable usage to constants change client cache from json to h2 mvstore database and a couple of fixes and performance improvements --- core/build.gradle.kts | 3 +- .../{GlobalVariables.java => Constants.java} | 5 +- .../pl/skidam/automodpack_core/Server.java | 3 +- .../skidam/automodpack_core/auth/Secrets.java | 2 +- .../automodpack_core/auth/SecretsStore.java | 6 +- .../automodpack_core/config/ConfigTools.java | 2 +- .../automodpack_core/config/ConfigUtils.java | 2 +- .../loader/LoaderManagerService.java | 5 - .../loader/ModpackLoaderService.java | 3 +- .../loader/NullLoaderManager.java | 9 - .../loader/NullModpackLoader.java | 3 +- .../modpack/ModpackContent.java | 18 +- .../modpack/ModpackExecutor.java | 4 +- .../protocol/DownloadClient.java | 88 ++-- .../automodpack_core/protocol/NetUtils.java | 6 +- .../compression/CompressionCodec.java | 28 +- .../protocol/compression/GzipCompression.java | 9 +- .../protocol/compression/NoneCompression.java | 11 +- .../protocol/compression/ZstdCompression.java | 23 +- .../protocol/netty/NettyServer.java | 8 +- .../protocol/netty/TrafficShaper.java | 4 +- .../netty/handler/ConfigurationHandler.java | 2 +- .../protocol/netty/handler/ErrorPrinter.java | 2 +- .../netty/handler/ProtocolServerHandler.java | 6 +- .../netty/handler/ServerMessageHandler.java | 14 +- .../utils/AddressHelpers.java | 2 +- .../utils/FileInspection.java | 25 +- .../utils/FileTreeScanner.java | 2 +- .../automodpack_core/utils/JarUtils.java | 2 +- .../skidam/automodpack_core/utils/Json.java | 4 +- ...Utils.java => LegacyClientCacheUtils.java} | 83 +--- .../utils/LockFreeInputStream.java | 2 +- .../utils/ModpackContentTools.java | 2 +- .../automodpack_core/utils/ObservableMap.java | 9 +- .../automodpack_core/utils/PlatformUtils.java | 2 +- ...stomFileUtils.java => SmartFileUtils.java} | 142 ++++--- .../utils/WorkaroundUtil.java | 6 +- .../utils/cache/FileMetadataCache.java | 141 +++++++ .../launchers/LauncherVersionSwapper.java | 2 +- .../utils/launchers/MultiMCMeta.java | 10 +- .../utils/launchers/PandoraMeta.java | 4 +- .../automodpack_core/modpack/ModpackTest.java | 9 +- ...UtilsTest.java => SmartFileUtilsTest.java} | 12 +- .../skidam/automodpack_loader_core/Gui.java | 4 +- .../automodpack_loader_core/Preload.java | 11 +- .../automodpack_loader_core/ReLauncher.java | 2 +- .../automodpack_loader_core/SelfUpdater.java | 12 +- .../client/ModpackUpdater.java | 391 ++++++++++-------- .../client/ModpackUtils.java | 149 +++---- .../crashassistant/ProcessSignalIO.java | 12 +- .../loader/LoaderManager.java | 8 - .../mods/ModpackLoader.java | 3 +- .../platforms/CurseForgeAPI.java | 2 +- .../platforms/ModrinthAPI.java | 2 +- .../utils/DownloadManager.java | 38 +- .../utils/FetchManager.java | 2 +- .../mods/ModpackLoader15.java | 10 +- .../mods/ModpackLoader16.java | 8 +- .../loader/LoaderManager.java | 72 ---- .../mods/ModpackLoader.java | 7 +- .../loader/LoaderManager.java | 80 +--- .../mods/ModpackLoader.java | 5 +- .../loader/LoaderManager.java | 80 +--- .../mods/ModpackLoader.java | 5 +- loader/loader-fabric-core.gradle.kts | 1 + loader/loader-forge.gradle.kts | 1 + loader/loader-neoforge.gradle.kts | 1 + .../loader/LoaderManager.java | 81 +--- .../mods/ModpackLoader.java | 5 +- .../loader/LoaderManager.java | 81 +--- .../mods/ModpackLoader.java | 5 +- .../loader/LoaderManager.java | 81 +--- .../mods/ModpackLoader.java | 5 +- .../client/audio/AudioManager.java | 4 +- .../automodpack/client/ui/DownloadScreen.java | 4 +- .../ui/FingerprintVerificationScreen.java | 4 +- .../client/ui/SkipVerificationScreen.java | 4 +- .../pl/skidam/automodpack/init/Common.java | 2 +- .../skidam/automodpack/init/FabricInit.java | 2 +- .../pl/skidam/automodpack/init/ForgeInit.java | 2 +- .../skidam/automodpack/init/NeoForgeInit.java | 2 +- .../skidam/automodpack/loader/GameCall.java | 2 +- .../core/LoginQueryRequestS2CPacketMixin.java | 4 +- .../mixin/core/PlayerManagerMixin.java | 2 +- .../core/ServerLoginNetworkHandlerMixin.java | 2 +- .../mixin/core/ServerNetworkIoMixin.java | 6 +- .../automodpack/mixin/dev/TestButton.java | 2 +- .../skidam/automodpack/modpack/Commands.java | 2 +- .../networking/LoginNetworkingIDs.java | 2 +- .../networking/LoginQueryParser.java | 2 +- .../automodpack/networking/ModPackets.java | 2 +- .../client/ClientLoginNetworkAddon.java | 2 +- .../networking/packet/DataC2SPacket.java | 2 +- .../networking/packet/DataS2CPacket.java | 2 +- .../networking/packet/HandshakeC2SPacket.java | 2 +- .../networking/packet/HandshakeS2CPacket.java | 2 +- .../server/ServerLoginNetworkAddon.java | 2 +- 97 files changed, 825 insertions(+), 1126 deletions(-) rename core/src/main/java/pl/skidam/automodpack_core/{GlobalVariables.java => Constants.java} (96%) rename core/src/main/java/pl/skidam/automodpack_core/utils/{ClientCacheUtils.java => LegacyClientCacheUtils.java} (52%) rename core/src/main/java/pl/skidam/automodpack_core/utils/{CustomFileUtils.java => SmartFileUtils.java} (64%) create mode 100644 core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java rename core/src/test/java/pl/skidam/automodpack_core/utils/{CustomFileUtilsTest.java => SmartFileUtilsTest.java} (82%) diff --git a/core/build.gradle.kts b/core/build.gradle.kts index 78d4d4a73..a1ee153f8 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -21,7 +21,8 @@ val deps = listOf( "com.google.code.gson:gson:2.13.2", "org.bouncycastle:bcpkix-jdk18on:1.82", "org.apache.httpcomponents.client5:httpclient5:5.5.1", - "org.tomlj:tomlj:1.1.1" + "org.tomlj:tomlj:1.1.1", + "com.h2database:h2-mvstore:2.4.240" ) dependencies { diff --git a/core/src/main/java/pl/skidam/automodpack_core/GlobalVariables.java b/core/src/main/java/pl/skidam/automodpack_core/Constants.java similarity index 96% rename from core/src/main/java/pl/skidam/automodpack_core/GlobalVariables.java rename to core/src/main/java/pl/skidam/automodpack_core/Constants.java index f28ded8ee..c61d03771 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/GlobalVariables.java +++ b/core/src/main/java/pl/skidam/automodpack_core/Constants.java @@ -9,7 +9,9 @@ import java.nio.file.Path; -public class GlobalVariables { +// More or less constants +// TODO cleanup +public class Constants { public static final Logger LOGGER = LogManager.getLogger("AutoModpack"); public static final String MOD_ID = "automodpack"; // For real its "automodpack_mod" but we use this for resource locations etc. public static Boolean DEBUG = false; @@ -39,6 +41,7 @@ public class GlobalVariables { public static Path hostModpackContentFile = hostModpackDir.resolve("automodpack-content.json"); public static Path serverConfigFile = automodpackDir.resolve("automodpack-server.json"); public static Path clientLocalMetadataFile = automodpackDir.resolve("automodpack-client-metadata.json"); + public static Path hashCacheDBFile = automodpackDir.resolve("hash-cache.db"); public static Path clientDummyFilesFile = automodpackDir.resolve("automodpack-dummy-files.json"); public static Path clientDeletionTimeStamps = automodpackDir.resolve("automodpack-deletion-timestamps-files.json"); public static Path serverCoreConfigFile = automodpackDir.resolve("automodpack-core.json"); diff --git a/core/src/main/java/pl/skidam/automodpack_core/Server.java b/core/src/main/java/pl/skidam/automodpack_core/Server.java index 4f776783d..a1993b433 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/Server.java +++ b/core/src/main/java/pl/skidam/automodpack_core/Server.java @@ -7,10 +7,9 @@ import pl.skidam.automodpack_core.protocol.netty.NettyServer; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashSet; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class Server { diff --git a/core/src/main/java/pl/skidam/automodpack_core/auth/Secrets.java b/core/src/main/java/pl/skidam/automodpack_core/auth/Secrets.java index 1a43852cb..3d818439d 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/auth/Secrets.java +++ b/core/src/main/java/pl/skidam/automodpack_core/auth/Secrets.java @@ -6,7 +6,7 @@ import java.security.SecureRandom; import java.util.Base64; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class Secrets { public static class Secret { // unfortunately has to be a class instead of record because of older gson version in 1.18 mc diff --git a/core/src/main/java/pl/skidam/automodpack_core/auth/SecretsStore.java b/core/src/main/java/pl/skidam/automodpack_core/auth/SecretsStore.java index fbe47bc84..38fd3843b 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/auth/SecretsStore.java +++ b/core/src/main/java/pl/skidam/automodpack_core/auth/SecretsStore.java @@ -1,6 +1,6 @@ package pl.skidam.automodpack_core.auth; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.config.ConfigTools; import pl.skidam.automodpack_core.config.Jsons; @@ -52,8 +52,8 @@ public void save(String key, Secrets.Secret secret) throws IllegalArgumentExcept } } - private static final SecretsCache hostSecrets = new SecretsCache(GlobalVariables.serverSecretsFile); - private static final SecretsCache clientSecrets = new SecretsCache(GlobalVariables.clientSecretsFile); + private static final SecretsCache hostSecrets = new SecretsCache(Constants.serverSecretsFile); + private static final SecretsCache clientSecrets = new SecretsCache(Constants.clientSecretsFile); public static Map.Entry getHostSecret(String secret) { hostSecrets.load(); diff --git a/core/src/main/java/pl/skidam/automodpack_core/config/ConfigTools.java b/core/src/main/java/pl/skidam/automodpack_core/config/ConfigTools.java index 7f921017e..359b8c003 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/config/ConfigTools.java +++ b/core/src/main/java/pl/skidam/automodpack_core/config/ConfigTools.java @@ -10,7 +10,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class ConfigTools { diff --git a/core/src/main/java/pl/skidam/automodpack_core/config/ConfigUtils.java b/core/src/main/java/pl/skidam/automodpack_core/config/ConfigUtils.java index 8a3c7e23e..11b7f58be 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/config/ConfigUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/config/ConfigUtils.java @@ -3,7 +3,7 @@ import java.util.*; import java.util.regex.Pattern; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class ConfigUtils { diff --git a/core/src/main/java/pl/skidam/automodpack_core/loader/LoaderManagerService.java b/core/src/main/java/pl/skidam/automodpack_core/loader/LoaderManagerService.java index 78b2ac6a4..5f2b221b8 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/loader/LoaderManagerService.java +++ b/core/src/main/java/pl/skidam/automodpack_core/loader/LoaderManagerService.java @@ -1,15 +1,10 @@ package pl.skidam.automodpack_core.loader; -import pl.skidam.automodpack_core.utils.FileInspection; - -import java.util.Collection; - public interface LoaderManagerService { enum ModPlatform { FABRIC, QUILT, FORGE, NEOFORGE } enum EnvironmentType { CLIENT, SERVER, UNIVERSAL } ModPlatform getPlatformType(); - Collection getModList(); boolean isModLoaded(String modId); String getLoaderVersion(); EnvironmentType getEnvironmentType(); diff --git a/core/src/main/java/pl/skidam/automodpack_core/loader/ModpackLoaderService.java b/core/src/main/java/pl/skidam/automodpack_core/loader/ModpackLoaderService.java index c8a98183c..9d4531294 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/loader/ModpackLoaderService.java +++ b/core/src/main/java/pl/skidam/automodpack_core/loader/ModpackLoaderService.java @@ -1,11 +1,12 @@ package pl.skidam.automodpack_core.loader; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.List; public interface ModpackLoaderService { void loadModpack(List modpackMods); - List getModpackNestedConflicts(Path modpackDir); // Returns list of mods from the modpack Dir that are conflicting with the mods from standard mods dir + List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache); // Returns list of mods from the modpack Dir that are conflicting with the mods from standard mods dir } diff --git a/core/src/main/java/pl/skidam/automodpack_core/loader/NullLoaderManager.java b/core/src/main/java/pl/skidam/automodpack_core/loader/NullLoaderManager.java index 8adb15881..340def168 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/loader/NullLoaderManager.java +++ b/core/src/main/java/pl/skidam/automodpack_core/loader/NullLoaderManager.java @@ -1,9 +1,5 @@ package pl.skidam.automodpack_core.loader; -import pl.skidam.automodpack_core.utils.FileInspection; - -import java.util.Collection; - public class NullLoaderManager implements LoaderManagerService { @Override public ModPlatform getPlatformType() { @@ -15,11 +11,6 @@ public boolean isModLoaded(String modId) { return false; } - @Override - public Collection getModList() { - return null; - } - @Override public String getLoaderVersion() { return null; diff --git a/core/src/main/java/pl/skidam/automodpack_core/loader/NullModpackLoader.java b/core/src/main/java/pl/skidam/automodpack_core/loader/NullModpackLoader.java index 47d3c1fb1..fd840d9c8 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/loader/NullModpackLoader.java +++ b/core/src/main/java/pl/skidam/automodpack_core/loader/NullModpackLoader.java @@ -1,6 +1,7 @@ package pl.skidam.automodpack_core.loader; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.List; @@ -13,7 +14,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { throw new AssertionError("Loader class not found"); } } diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java index 50912cfe2..ae37c1c87 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java @@ -13,8 +13,8 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; -import static pl.skidam.automodpack_core.GlobalVariables.*; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.*; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class ModpackContent { public final Set list = ConcurrentHashMap.newKeySet(); @@ -78,7 +78,7 @@ public boolean create() { previousContent.list.forEach(item -> sha1MurmurMapPreviousContent.put(item.sha1, item.murmur)); }); - List> creationFutures = Collections.synchronizedList(new ArrayList<>()); + List> creationFutures = new ArrayList<>(); // host-modpack generation if (MODPACK_DIR != null) { @@ -131,8 +131,8 @@ public boolean loadPreviousContent() { list.addAll(previousModpackContent.list); for (Jsons.ModpackContentFields.ModpackContentItem modpackContentItem : list) { - Path file = CustomFileUtils.getPath(MODPACK_DIR, modpackContentItem.file); - if (!Files.exists(file)) file = CustomFileUtils.getPathFromCWD(modpackContentItem.file); + Path file = SmartFileUtils.getPath(MODPACK_DIR, modpackContentItem.file); + if (!Files.exists(file)) file = SmartFileUtils.getPathFromCWD(modpackContentItem.file); if (!Files.exists(file)) { LOGGER.warn("File {} does not exist!", file); continue; @@ -209,7 +209,7 @@ public void replace(Path file) { public void remove(Path file) { - String modpackFile = CustomFileUtils.formatPath(file, MODPACK_DIR); + String modpackFile = SmartFileUtils.formatPath(file, MODPACK_DIR); synchronized (list) { for (Jsons.ModpackContentFields.ModpackContentItem item : this.list) { @@ -247,7 +247,7 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path return null; } - String formattedFile = CustomFileUtils.formatPath(file, MODPACK_DIR); + String formattedFile = SmartFileUtils.formatPath(file, MODPACK_DIR); // modpackFile is relative path to ~/.minecraft/ (content format) so if it starts with /automodpack/ we dont want it if (formattedFile.startsWith("/automodpack/")) { @@ -308,7 +308,7 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path type = "other"; } - String sha1 = CustomFileUtils.getHash(file); + String sha1 = SmartFileUtils.getHash(file); // For CF API String murmur = null; @@ -316,7 +316,7 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path // get murmur hash from previousContent.list of item with same sha1 murmur = sha1MurmurMapPreviousContent.get(sha1); if (murmur == null) { - murmur = CustomFileUtils.getCurseforgeMurmurHash(file); + murmur = SmartFileUtils.getCurseforgeMurmurHash(file); } } diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java index 887dd7aeb..fdea654bb 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java @@ -8,11 +8,11 @@ import java.util.*; import java.util.concurrent.*; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class ModpackExecutor { private final ThreadPoolExecutor CREATION_EXECUTOR = (ThreadPoolExecutor) Executors.newFixedThreadPool(Math.max(1, Runtime.getRuntime().availableProcessors() * 2), new CustomThreadFactoryBuilder().setNameFormat("AutoModpackCreation-%d").build()); - public final Map modpacks = Collections.synchronizedMap(new HashMap<>()); + public final Map modpacks = new ConcurrentHashMap<>(); private ModpackContent init() { if (isGenerating()) { diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/DownloadClient.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/DownloadClient.java index d485430b3..36bd6ba4f 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/DownloadClient.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/DownloadClient.java @@ -1,11 +1,12 @@ package pl.skidam.automodpack_core.protocol; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import static pl.skidam.automodpack_core.protocol.NetUtils.*; import java.io.*; import java.net.InetSocketAddress; import java.net.Socket; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -291,6 +292,10 @@ class Connection implements AutoCloseable { private final ExecutorService executor = Executors.newSingleThreadExecutor(); private final AtomicBoolean busy = new AtomicBoolean(false); + // Reuse this buffer for reading from socket to avoid allocation per frame. + // Size = Default Chunk + Header overhead (approx) + Safety margin + private final byte[] networkInputBuffer = new byte[MAX_CHUNK_SIZE + 8192]; + public Connection(PreValidationConnection preValidationConnection, byte[] secretBytes) throws IOException { if (preValidationConnection.getSocket() == null || preValidationConnection.getSocket().isClosed()) { throw new SSLHandshakeException("Server certificate invalid, connection closed"); @@ -439,61 +444,62 @@ private byte[] readProtocolMessageFrame() throws IOException { throw new IOException("Frame original length (" + originalLength + ") exceeds chunk size (" + this.chunkSize + ")"); } - byte[] compressed = new byte[compressedLength]; - in.readFully(compressed); + if (compressedLength > networkInputBuffer.length) { + throw new IOException("Compressed length exceeds buffer capacity"); + } + + in.readFully(networkInputBuffer, 0, compressedLength); - return getCompressionCodec().decompress(compressed, originalLength); + return getCompressionCodec().decompress(networkInputBuffer, 0, compressedLength, originalLength); } /** * Processes the server response stream. Expects Header -> Data Frames -> EOT. */ private Path readFileResponse(Path destination, IntConsumer chunkCallback) throws IOException { - try (DataInputStream headerIn = new DataInputStream(new ByteArrayInputStream(readProtocolMessageFrame()))) { - byte version = headerIn.readByte(); - byte messageType = headerIn.readByte(); - - if (messageType == ERROR) { - int errLen = headerIn.readInt(); - byte[] errBytes = new byte[errLen]; - headerIn.readFully(errBytes); - throw new IOException("Server error: " + new String(errBytes, StandardCharsets.UTF_8)); - } + byte[] headerData = readProtocolMessageFrame(); + ByteBuffer headerWrap = ByteBuffer.wrap(headerData); - if (messageType == END_OF_TRANSMISSION) { - return destination; - } + byte version = headerWrap.get(); + byte messageType = headerWrap.get(); - if (messageType != FILE_RESPONSE_TYPE) { - throw new IOException("Unexpected message type: " + messageType); - } + if (messageType == ERROR) { + int errLen = headerWrap.getInt(); + byte[] errBytes = new byte[errLen]; + headerWrap.get(errBytes); + throw new IOException("Server error: " + new String(errBytes, StandardCharsets.UTF_8)); + } - long expectedFileSize = headerIn.readLong(); - long receivedBytes = 0; + if (messageType == END_OF_TRANSMISSION) { + return destination; + } - try (OutputStream fos = new BufferedOutputStream(Files.newOutputStream(destination, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE))) { + if (messageType != FILE_RESPONSE_TYPE) { + throw new IOException("Unexpected message type: " + messageType); + } - while (receivedBytes < expectedFileSize) { - byte[] dataFrame = readProtocolMessageFrame(); - int toWrite = Math.min(dataFrame.length, (int) (expectedFileSize - receivedBytes)); - fos.write(dataFrame, 0, toWrite); - receivedBytes += toWrite; - if (chunkCallback != null) chunkCallback.accept(toWrite); - } - } + long expectedFileSize = headerWrap.getLong(); + long receivedBytes = 0; - try (DataInputStream eotIn = new DataInputStream(new ByteArrayInputStream(readProtocolMessageFrame()))) { - byte ver = eotIn.readByte(); - byte eotType = eotIn.readByte(); - if (ver != version || eotType != END_OF_TRANSMISSION) { - throw new IOException("Invalid EOT frame"); - } + try (OutputStream fos = new BufferedOutputStream(Files.newOutputStream(destination, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE))) { + + while (receivedBytes < expectedFileSize) { + byte[] dataFrame = readProtocolMessageFrame(); + int toWrite = Math.min(dataFrame.length, (int) (expectedFileSize - receivedBytes)); + fos.write(dataFrame, 0, toWrite); + receivedBytes += toWrite; + if (chunkCallback != null) chunkCallback.accept(toWrite); } - return destination; } + + byte[] eotData = readProtocolMessageFrame(); + if (eotData.length < 2 || eotData[0] != version || eotData[1] != END_OF_TRANSMISSION) { + throw new IOException("Invalid EOT frame"); + } + return destination; } private byte sendCompressionConfig(byte desiredCompression) throws IOException { diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java index 137ca0865..cf5cd80ab 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java @@ -6,7 +6,7 @@ import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.operator.ContentSigner; import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; -import pl.skidam.automodpack_core.utils.CustomFileUtils; +import pl.skidam.automodpack_core.utils.SmartFileUtils; import pl.skidam.automodpack_core.utils.LockFreeInputStream; import java.io.InputStream; @@ -100,7 +100,7 @@ public static void saveCertificate(X509Certificate cert, Path path) throws Excep String certPem = "-----BEGIN CERTIFICATE-----\n" + formatBase64(Base64.getEncoder().encodeToString(cert.getEncoded())) + "-----END CERTIFICATE-----"; - CustomFileUtils.setupFilePaths(path); + SmartFileUtils.setupFilePaths(path); Files.writeString(path, certPem); } @@ -117,7 +117,7 @@ public static void savePrivateKey(PrivateKey key, Path path) throws Exception { String keyPem = "-----BEGIN PRIVATE KEY-----\n" + formatBase64(Base64.getEncoder().encodeToString(keySpec.getEncoded())) + "-----END PRIVATE KEY-----"; - CustomFileUtils.setupFilePaths(path); + SmartFileUtils.setupFilePaths(path); Files.writeString(path, keyPem); } diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/CompressionCodec.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/CompressionCodec.java index 1c0161785..ab8bb5a92 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/CompressionCodec.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/CompressionCodec.java @@ -21,6 +21,8 @@ public interface CompressionCodec { /** * Decompresses the compressed data. + *

+ * Note: This legacy method assumes the entire array is the compressed payload. * * @param compressed the compressed data * @param originalLength the expected length of the decompressed data @@ -29,10 +31,34 @@ public interface CompressionCodec { */ byte[] decompress(byte[] compressed, int originalLength) throws IOException; + /** + * Decompresses a specific range of the compressed data buffer. + *

+ * This method allows zero-copy processing of input buffers (e.g. reused network buffers). + * + * @param compressedBuffer the buffer containing compressed data + * @param offset the start offset of the compressed data + * @param length the length of the compressed data + * @param originalLength the expected length of the decompressed data + * @return the decompressed data + * @throws IOException if decompression fails + */ + default byte[] decompress(byte[] compressedBuffer, int offset, int length, int originalLength) throws IOException { + // Default implementation for backward compatibility or simple codecs: + // Create a slice and delegate to the simple method. + // Subclasses (GZIP/Zstd) should override this to avoid the copy. + if (offset == 0 && length == compressedBuffer.length) { + return decompress(compressedBuffer, originalLength); + } + byte[] slice = new byte[length]; + System.arraycopy(compressedBuffer, offset, slice, 0, length); + return decompress(slice, originalLength); + } + /** * Gets the compression type identifier for this codec. * * @return the compression type constant from NetUtils */ byte getCompressionType(); -} +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/GzipCompression.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/GzipCompression.java index 6162a32ca..8ca1adbf8 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/GzipCompression.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/GzipCompression.java @@ -32,7 +32,12 @@ public byte[] compress(byte[] input) throws IOException { @Override public byte[] decompress(byte[] compressed, int originalLength) throws IOException { - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressed); + return decompress(compressed, 0, compressed.length, originalLength); + } + + @Override + public byte[] decompress(byte[] compressedBuffer, int offset, int length, int originalLength) throws IOException { + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedBuffer, offset, length); GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream)) { byte[] decompressed = new byte[originalLength]; @@ -57,4 +62,4 @@ public byte[] decompress(byte[] compressed, int originalLength) throws IOExcepti public byte getCompressionType() { return COMPRESSION_GZIP; } -} +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/NoneCompression.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/NoneCompression.java index 9f1d2d199..29d844a72 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/NoneCompression.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/NoneCompression.java @@ -1,5 +1,7 @@ package pl.skidam.automodpack_core.protocol.compression; +import java.util.Arrays; + import static pl.skidam.automodpack_core.protocol.NetUtils.COMPRESSION_NONE; /** @@ -23,8 +25,15 @@ public byte[] decompress(byte[] compressed, int originalLength) { return compressed; } + @Override + public byte[] decompress(byte[] compressedBuffer, int offset, int length, int originalLength) { + // For "None", we just return the slice of the buffer. + // We must copy it because the caller expects a standalone array of size 'originalLength'. + return Arrays.copyOfRange(compressedBuffer, offset, offset + length); + } + @Override public byte getCompressionType() { return COMPRESSION_NONE; } -} +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/ZstdCompression.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/ZstdCompression.java index 35555ab62..aac48d8c5 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/ZstdCompression.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/compression/ZstdCompression.java @@ -11,7 +11,7 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; import static pl.skidam.automodpack_core.protocol.NetUtils.COMPRESSION_ZSTD; /** @@ -22,6 +22,7 @@ public class ZstdCompression implements CompressionCodec { private static MethodHandle compressMethodHandle; private static MethodHandle decompressMethodHandle; + private static MethodHandle decompressFrameMethodHandle; private static boolean initialized = true; static { @@ -36,13 +37,15 @@ public class ZstdCompression implements CompressionCodec { tempJar.toFile().deleteOnExit(); URLClassLoader loader = new URLClassLoader(new URL[]{tempJar.toUri().toURL()}, ZstdCompression.class.getClassLoader()); - Class zstdClass = Class.forName("com.github.luben.zstd.Zstd", true, loader); + Method compressMethod = zstdClass.getMethod("compress", byte[].class); Method decompressMethod = zstdClass.getMethod("decompress", byte[].class, int.class); + Method decompressFrameMethod = zstdClass.getMethod("decompressFrame", byte[].class, int.class, int.class, int.class); compressMethodHandle = MethodHandles.lookup().unreflect(compressMethod); decompressMethodHandle = MethodHandles.lookup().unreflect(decompressMethod); + decompressFrameMethodHandle = MethodHandles.lookup().unreflect(decompressFrameMethod); } catch (Throwable e) { initialized = false; LOGGER.debug("Failed to initialize embedded zstd-jni", e); @@ -76,8 +79,22 @@ public byte[] decompress(byte[] compressed, int originalLength) throws IOExcepti } } + @Override + public byte[] decompress(byte[] compressedBuffer, int offset, int length, int originalLength) throws IOException { + try { + // Signature: byte[] src, int srcOffset, int srcSize, int originalSize + byte[] decompressed = (byte[]) decompressFrameMethodHandle.invokeExact(compressedBuffer, offset, length, originalLength); + if (decompressed.length != originalLength) { + throw new IOException("Unexpected decompressed length: " + decompressed.length + " (expected " + originalLength + ")"); + } + return decompressed; + } catch (Throwable e) { + throw new IOException("Zstd range decompression failed", e); + } + } + @Override public byte getCompressionType() { return COMPRESSION_ZSTD; } -} +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java index be6f7c6fc..095279b3c 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java @@ -1,6 +1,6 @@ package pl.skidam.automodpack_core.protocol.netty; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.*; @@ -21,6 +21,8 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + import pl.skidam.automodpack_core.config.ConfigTools; import pl.skidam.automodpack_core.protocol.NetUtils; import pl.skidam.automodpack_core.protocol.netty.handler.ProtocolServerHandler; @@ -34,8 +36,8 @@ public class NettyServer { public static final AttributeKey COMPRESSION_TYPE = AttributeKey.valueOf("COMPRESSION_TYPE"); public static final AttributeKey CHUNK_SIZE = AttributeKey.valueOf("CHUNK_SIZE"); public static final AttributeKey PROTOCOL_VERSION = AttributeKey.valueOf("PROTOCOL_VERSION"); - private final Map connections = Collections.synchronizedMap(new HashMap<>()); - private final Map paths = Collections.synchronizedMap(new HashMap<>()); + private final Map connections = new ConcurrentHashMap<>(); + private final Map paths = new ConcurrentHashMap<>(); private MultithreadEventLoopGroup eventLoopGroup; private ChannelFuture serverChannel; private Boolean shouldHost = false; // needed for stop modpack hosting for minecraft port diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/TrafficShaper.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/TrafficShaper.java index 75e5315b3..49c2b4777 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/TrafficShaper.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/TrafficShaper.java @@ -5,8 +5,8 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; -import static pl.skidam.automodpack_core.GlobalVariables.serverConfig; +import static pl.skidam.automodpack_core.Constants.LOGGER; +import static pl.skidam.automodpack_core.Constants.serverConfig; public class TrafficShaper { diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ConfigurationHandler.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ConfigurationHandler.java index 333d6cbc6..f566be92f 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ConfigurationHandler.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ConfigurationHandler.java @@ -1,6 +1,6 @@ package pl.skidam.automodpack_core.protocol.netty.handler; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; import static pl.skidam.automodpack_core.protocol.NetUtils.*; import io.netty.buffer.ByteBuf; diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ErrorPrinter.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ErrorPrinter.java index 5f2da1aa0..90dfe1e19 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ErrorPrinter.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ErrorPrinter.java @@ -7,7 +7,7 @@ import javax.net.ssl.SSLHandshakeException; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; /** * A fallback error handler that logs caught exceptions. In order to reduce verbosity, TLS handshake errors are printed diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ProtocolServerHandler.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ProtocolServerHandler.java index d3d4b86d5..7c841b82b 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ProtocolServerHandler.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ProtocolServerHandler.java @@ -18,9 +18,9 @@ import java.net.SocketAddress; import java.util.List; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; -import static pl.skidam.automodpack_core.GlobalVariables.hostServer; -import static pl.skidam.automodpack_core.GlobalVariables.serverConfig; +import static pl.skidam.automodpack_core.Constants.LOGGER; +import static pl.skidam.automodpack_core.Constants.hostServer; +import static pl.skidam.automodpack_core.Constants.serverConfig; import static pl.skidam.automodpack_core.protocol.NetUtils.*; public class ProtocolServerHandler extends ByteToMessageDecoder { diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java index 21d12ceb3..97796904a 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java @@ -1,10 +1,9 @@ package pl.skidam.automodpack_core.protocol.netty.handler; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import static pl.skidam.automodpack_core.protocol.NetUtils.*; import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; @@ -56,7 +55,7 @@ protected void channelRead0(ChannelHandlerContext ctx, ProtocolMessage msg) thro switch (msg.getType()) { case ECHO_TYPE: EchoMessage echoMsg = (EchoMessage) msg; - ByteBuf echoBuf = Unpooled.buffer(1 + 1 + msg.getSecret().length + echoMsg.getData().length); + ByteBuf echoBuf = ctx.alloc().buffer(1 + 1 + msg.getSecret().length + echoMsg.getData().length); echoBuf.writeByte(clientProtocolVersion); echoBuf.writeByte(ECHO_TYPE); echoBuf.writeBytes(echoMsg.getSecret()); @@ -83,7 +82,7 @@ private void refreshModpackFiles(ChannelHandlerContext context, byte[][] FileHas hashes.add(new String(hash)); } LOGGER.info("Received refresh request for files of hashes: {}", hashes); - Set> creationFutures = new HashSet<>(); + List> creationFutures = new ArrayList<>(); Set modpacks = new HashSet<>(); for (String hash : hashes) { @@ -156,8 +155,7 @@ private void sendFile(ChannelHandlerContext ctx, byte[] bsha1) throws IOExceptio final Path path = optionalPath.get(); final long fileSize = Files.size(path); - // Send file response header: version, FILE_RESPONSE type, then file size (8 bytes) - ByteBuf responseHeader = Unpooled.buffer(1 + 1 + 8); + ByteBuf responseHeader = ctx.alloc().buffer(1 + 1 + 8); responseHeader.writeByte(this.protocolVersion); responseHeader.writeByte(FILE_RESPONSE_TYPE); responseHeader.writeLong(fileSize); @@ -204,7 +202,7 @@ public Optional resolvePath(final String sha1) { private void sendError(ChannelHandlerContext ctx, byte version, String errorMessage) { byte[] errMsgBytes = errorMessage.getBytes(CharsetUtil.UTF_8); - ByteBuf errorBuf = Unpooled.buffer(1 + 1 + 4 + errMsgBytes.length); + ByteBuf errorBuf = ctx.alloc().buffer(1 + 1 + 4 + errMsgBytes.length); errorBuf.writeByte(version); errorBuf.writeByte(ERROR); errorBuf.writeInt(errMsgBytes.length); @@ -214,7 +212,7 @@ private void sendError(ChannelHandlerContext ctx, byte version, String errorMess } private void sendEOT(ChannelHandlerContext ctx) { - ByteBuf eot = Unpooled.buffer(2); + ByteBuf eot = ctx.alloc().buffer(2); eot.writeByte(this.protocolVersion); eot.writeByte(END_OF_TRANSMISSION); ctx.writeAndFlush(eot); diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/AddressHelpers.java b/core/src/main/java/pl/skidam/automodpack_core/utils/AddressHelpers.java index 28f19dd66..c0088ce1f 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/AddressHelpers.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/AddressHelpers.java @@ -4,7 +4,7 @@ import java.util.Enumeration; import java.util.Objects; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class AddressHelpers { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java index 8a0bbb99a..f42053f57 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java @@ -7,7 +7,7 @@ import org.tomlj.TomlArray; import org.tomlj.TomlParseResult; import org.tomlj.TomlTable; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.loader.LoaderManagerService; import java.io.BufferedReader; @@ -23,12 +23,12 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class FileInspection { private static final Gson GSON = new Gson(); - private static final String LOADER = GlobalVariables.LOADER; + private static final String LOADER = Constants.LOADER; public static boolean isMod(Path file) { if (!file.getFileName().toString().endsWith(".jar") || !Files.exists(file)) { @@ -44,30 +44,17 @@ public static boolean isMod(Path file) { public record Mod(String modID, String hash, Collection providesIDs, String modVersion, Path modPath, LoaderManagerService.EnvironmentType environmentType, Collection dependencies) {} public record HashPathPair(String hash, Path path) { } - private static final Map modCache = new HashMap<>(); public static Mod getMod(Path file) { if (!Files.isRegularFile(file)) return null; if (!file.getFileName().toString().endsWith(".jar")) return null; - String hash = CustomFileUtils.getHash(file); + String hash = SmartFileUtils.getHash(file); if (hash == null) { LOGGER.error("Failed to get hash for file: {}", file); return null; } - HashPathPair hashPathPair = new HashPathPair(hash, file); - if (modCache.containsKey(hashPathPair)) { - return modCache.get(hashPathPair); - } - - for (Mod mod : GlobalVariables.LOADER_MANAGER.getModList()) { - if (hash.equals(mod.hash)) { - modCache.put(hashPathPair, mod); - return mod; - } - } - // Open FS once for all metadata extractions try (FileSystem fs = FileSystems.newFileSystem(file)) { String modId = (String) getModInfo(fs, "modId"); @@ -79,9 +66,7 @@ public static Mod getMod(Path file) { Set providesIDs = getProvidedIDs(fs); if (modVersion != null && dependencies != null) { - var mod = new Mod(modId, hash, providesIDs, modVersion, file, environmentType, dependencies); - modCache.put(hashPathPair, mod); - return mod; + return new Mod(modId, hash, providesIDs, modVersion, file, environmentType, dependencies); } LOGGER.error("Not enough mod information for file: {} modId: {}, modVersion: {}, dependencies: {}", file, modId, modVersion, dependencies); diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/FileTreeScanner.java b/core/src/main/java/pl/skidam/automodpack_core/utils/FileTreeScanner.java index 742b2ef0b..286bfd3ec 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/FileTreeScanner.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/FileTreeScanner.java @@ -5,7 +5,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.*; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class FileTreeScanner { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/JarUtils.java b/core/src/main/java/pl/skidam/automodpack_core/utils/JarUtils.java index 2fade1e51..b70b5f3b8 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/JarUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/JarUtils.java @@ -4,7 +4,7 @@ import java.nio.file.Path; import java.security.CodeSource; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class JarUtils { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/Json.java b/core/src/main/java/pl/skidam/automodpack_core/utils/Json.java index ace6986d7..a3c188b74 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/Json.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/Json.java @@ -12,8 +12,8 @@ import java.nio.file.Path; import java.util.List; -import static pl.skidam.automodpack_core.GlobalVariables.AM_VERSION; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.AM_VERSION; +import static pl.skidam.automodpack_core.Constants.LOGGER; @SuppressWarnings("deprecation") public class Json { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/ClientCacheUtils.java b/core/src/main/java/pl/skidam/automodpack_core/utils/LegacyClientCacheUtils.java similarity index 52% rename from core/src/main/java/pl/skidam/automodpack_core/utils/ClientCacheUtils.java rename to core/src/main/java/pl/skidam/automodpack_core/utils/LegacyClientCacheUtils.java index bbb9e239a..93bf6bd96 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/ClientCacheUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/LegacyClientCacheUtils.java @@ -5,13 +5,11 @@ import java.io.FileOutputStream; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; -public class ClientCacheUtils { +public class LegacyClientCacheUtils { // TODO change this dummy byte array to contain also some metadata that we have created it private static final byte[] smallDummyJar = { @@ -28,80 +26,11 @@ public class ClientCacheUtils { 77, 70, -2, -54, 0, 0, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 70, 0, 0, 0, 97, 0, 0, 0, 0, 0, }; + private static final Jsons.ClientDummyFiles cacheDummyFiles = ConfigTools.load(clientDummyFilesFile, Jsons.ClientDummyFiles.class); - private static final Jsons.LocalMetadata clientMetadataCache = ConfigTools.load(clientLocalMetadataFile, Jsons.LocalMetadata.class); private static final Jsons.ClientDeletedNonModpackFilesTimestamps clientDeletedNonModpackFilesTimestamps = ConfigTools.load(clientDeletionTimeStamps, Jsons.ClientDeletedNonModpackFilesTimestamps.class); - // Metadata - - // Cheap cache check - 0 bytes read - public static String getVerifiedCacheHash(Path path) { - if (clientMetadataCache == null) return null; - - try { - BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); - Jsons.LocalMetadata.FileFingerprint fingerprint = clientMetadataCache.files.get(path.toAbsolutePath().normalize().toString()); - - if (fingerprint != null && fingerprint.lastSize == attrs.size() && fingerprint.lastModified == attrs.lastModifiedTime().toMillis()) { - return fingerprint.sha1; - } - } catch (IOException e) { - LOGGER.debug("Could not read attributes for {}", path); - } - return null; // Metadata mismatch or missing cache - } - - public static void updateMetadataCache(Path path, String hash) { - if (clientMetadataCache == null) return; - try { - BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); - clientMetadataCache.files.put(path.toAbsolutePath().normalize().toString(), new Jsons.LocalMetadata.FileFingerprint(hash, attrs.size(), attrs.lastModifiedTime().toMillis())); - } catch (IOException e) { - LOGGER.error("Could not update cache for {}", path, e); - } - } - - public static String computeHashIfNeeded(Path modPath) { - // Check cache first - String cachedHash = getVerifiedCacheHash(modPath); - if (cachedHash != null) return cachedHash; - - // Fallback to hashing and update cache - String diskHash = CustomFileUtils.getHash(modPath); - updateMetadataCache(modPath, diskHash); - return diskHash; - } - - public static boolean fastHashCompare(Path file1, Path file2) { - if (!Files.exists(file1) || !Files.exists(file2)) return false; - - String hash1 = computeHashIfNeeded(file1); - String hash2 = computeHashIfNeeded(file2); - - if (hash1 == null || hash2 == null) return false; - - return hash1.equals(hash2); - } - - public static void updateCache(Path path, String hash) { - if (clientMetadataCache == null) return; - try { - BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); - clientMetadataCache.files.put(path.toAbsolutePath().normalize().toString(), new Jsons.LocalMetadata.FileFingerprint(hash, attrs.size(), attrs.lastModifiedTime().toMillis())); - } catch (IOException e) { - LOGGER.error("Could not update cache for {}", path, e); - } - } - - public static void saveMetadataCache() { - if (clientMetadataCache == null) { - return; - } - ConfigTools.save(clientLocalMetadataFile, clientMetadataCache); - } - // Dummy - public static void deleteDummyFiles() { if (cacheDummyFiles == null || cacheDummyFiles.files.isEmpty()) { return; @@ -112,8 +41,8 @@ public static void deleteDummyFiles() { try { String filePath = iterator.next(); Path file = Path.of(filePath); - if (CustomFileUtils.compareFilesByteByByte(file, smallDummyJar)) { - CustomFileUtils.executeOrder66(file, false); + if (SmartFileUtils.compareSmallFile(file, smallDummyJar)) { + SmartFileUtils.executeOrder66(file, false); } iterator.remove(); } catch (Exception e) { @@ -147,6 +76,8 @@ public static void saveDummyFiles() { ConfigTools.save(clientDummyFilesFile, cacheDummyFiles); } + // Non Modpack Deleted files timestamps + public static boolean wasThisTimestampEvaluatedBefore(String timestamp) { if (clientDeletedNonModpackFilesTimestamps == null) return false; return clientDeletedNonModpackFilesTimestamps.timestamps.contains(timestamp); diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java b/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java index 9da45d314..870865711 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java @@ -9,7 +9,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import static pl.skidam.automodpack_core.utils.CustomFileUtils.isFilePhysical; +import static pl.skidam.automodpack_core.utils.SmartFileUtils.isFilePhysical; /** * A safe input stream that can read files currently locked or in-use by other processes (like active game logs). diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/ModpackContentTools.java b/core/src/main/java/pl/skidam/automodpack_core/utils/ModpackContentTools.java index bd24f32bc..388ee1163 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/ModpackContentTools.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/ModpackContentTools.java @@ -6,7 +6,7 @@ import java.nio.file.Path; import java.util.Optional; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class ModpackContentTools { public static String getFileType(String file, Jsons.ModpackContentFields list) { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/ObservableMap.java b/core/src/main/java/pl/skidam/automodpack_core/utils/ObservableMap.java index 98b63f4c7..e662476a6 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/ObservableMap.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/ObservableMap.java @@ -1,25 +1,26 @@ package pl.skidam.automodpack_core.utils; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.BiConsumer; @SuppressWarnings("unchecked") public class ObservableMap { - private final Map synchronizedMap; + private final ConcurrentHashMap synchronizedMap; private List> onPutCallbacks = new ArrayList<>(); private List> onRemoveCallbacks = new ArrayList<>() ; public ObservableMap() { - synchronizedMap = Collections.synchronizedMap(new HashMap<>()); + synchronizedMap = new ConcurrentHashMap<>(); } public ObservableMap(int initialCapacity) { - synchronizedMap = Collections.synchronizedMap(new HashMap<>(initialCapacity)); + synchronizedMap = new ConcurrentHashMap<>(initialCapacity); } public ObservableMap(Map m) { - synchronizedMap = Collections.synchronizedMap(new HashMap<>(m)); + synchronizedMap = new ConcurrentHashMap<>(m); } public synchronized V put(K key, V value) { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/PlatformUtils.java b/core/src/main/java/pl/skidam/automodpack_core/utils/PlatformUtils.java index bca13a586..010ff929a 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/PlatformUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/PlatformUtils.java @@ -3,7 +3,7 @@ import pl.skidam.automodpack_core.protocol.compression.CompressionCodec; import pl.skidam.automodpack_core.protocol.compression.CompressionFactory; import java.util.Locale; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; import static pl.skidam.automodpack_core.protocol.NetUtils.COMPRESSION_ZSTD; public class PlatformUtils { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/CustomFileUtils.java b/core/src/main/java/pl/skidam/automodpack_core/utils/SmartFileUtils.java similarity index 64% rename from core/src/main/java/pl/skidam/automodpack_core/utils/CustomFileUtils.java rename to core/src/main/java/pl/skidam/automodpack_core/utils/SmartFileUtils.java index bdacee597..e9468d596 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/CustomFileUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/SmartFileUtils.java @@ -5,14 +5,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.security.DigestInputStream; import java.security.MessageDigest; import java.util.*; import java.util.stream.Stream; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; -public class CustomFileUtils { +public class SmartFileUtils { public static final Path CWD = Path.of(System.getProperty("user.dir")); @@ -27,9 +26,9 @@ public static void executeOrder66(Path file, boolean saveDummyFiles) { } if (Files.isRegularFile(file)) { - ClientCacheUtils.dummyIT(file); + LegacyClientCacheUtils.dummyIT(file); if (saveDummyFiles) { - ClientCacheUtils.saveDummyFiles(); + LegacyClientCacheUtils.saveDummyFiles(); } } } @@ -86,24 +85,22 @@ public static void setupFilePaths(Path file) throws IOException { } } - public static boolean compareFilesByteByByte(Path path, byte[] referenceBytes) { + public static boolean compareSmallFile(Path path, byte[] referenceBytes) { try { if (Files.size(path) != referenceBytes.length) { return false; } - try (InputStream is = new BufferedInputStream(new LockFreeInputStream(path))) { - int b; - int i = 0; - while ((b = is.read()) != -1) { - if (b != (referenceBytes[i++] & 0xFF)) { // & 0xFF ensures unsigned comparison - return false; - } - } - return true; + try (InputStream is = new LockFreeInputStream(path)) { + // Java 11+ readNBytes reads exactly X bytes or until EOF. + // Since we know the file is small (~200b), reading it into RAM is perfectly fine. + byte[] fileContent = is.readNBytes(referenceBytes.length); + + // Vectorized Comparison (AVX optimized in Java 17) + return Arrays.equals(fileContent, referenceBytes); } } catch (Exception e) { - LOGGER.error("Error comparing file byte by byte: {}", path, e); + LOGGER.error("Error comparing file: {}", path, e); return false; } } @@ -150,13 +147,15 @@ public static String getHash(Path path) { try { MessageDigest digest = MessageDigest.getInstance("SHA-1"); - try (InputStream is = new BufferedInputStream(new LockFreeInputStream(path)); - DigestInputStream dis = new DigestInputStream(is, digest)) { - dis.transferTo(OutputStream.nullOutputStream()); // black hole :p + try (InputStream is = new LockFreeInputStream(path)) { + byte[] buffer = new byte[64 * 1024]; + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + digest.update(buffer, 0, bytesRead); + } } - byte[] hash = digest.digest(); - return HexFormat.of().formatHex(hash); + return HexFormat.of().formatHex(digest.digest()); } catch (IOException ignored) { // we don't really care about this exception, file may just not exists or be a directory } catch (Exception e) { LOGGER.error("Failed to get hash for path: {}", path, e); @@ -164,66 +163,81 @@ public static String getHash(Path path) { return null; } + // We do double pass to avoid storing whole file in memory public static String getCurseforgeMurmurHash(Path file) throws IOException { if (!Files.exists(file)) { return null; } - long length = 0; - - ByteArrayOutputStream filteredStream = new ByteArrayOutputStream(); - try (InputStream is = new BufferedInputStream(new LockFreeInputStream(file))) { - int b; - while ((b = is.read()) != -1) { - // Filter whitespace: Tab (0x9), LF (0xA), CR (0xD), Space (0x20) - if (b == 0x9 || b == 0xa || b == 0xd || b == 0x20) { - continue; + // MurmurHash2 Constants + final int m = 0x5bd1e995; + final int r = 24; + final int seed = 1; + + // Pass 1 + // We scan the file just to count non-whitespace bytes + long validLength = 0; + + byte[] buffer = new byte[64 * 1024]; + + try (InputStream is = new LockFreeInputStream(file)) { + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + for (int i = 0; i < bytesRead; i++) { + byte b = buffer[i]; + // Check for whitespace (Tab, LF, CR, Space) + if (b != 0x9 && b != 0xa && b != 0xd && b != 0x20) { + validLength++; + } } - - filteredStream.write(b); - length++; } } - // magic values - final int m = 0x5bd1e995; - final int r = 24; - long k = 0x0L; - int seed = 1; - int shift = 0x0; - char b; - long h = (seed ^ length); - - byte[] filteredBytes = filteredStream.toByteArray(); - - // Second pass: calculate hash using filtered bytes (no file I/O) - for (byte byteVal : filteredBytes) { - b = (char) (byteVal & 0xFF); - - k = k | ((long) b << shift); + // Pass 2 + // Now we have the length, we can initialize 'h' correctly with Bitwise XOR + long h = (seed ^ validLength); + long k = 0; + int shift = 0; + + try (InputStream is = new LockFreeInputStream(file)) { + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + for (int i = 0; i < bytesRead; i++) { + byte b = buffer[i]; + + // Same filter logic + if (b == 0x9 || b == 0xa || b == 0xd || b == 0x20) { + continue; + } - shift = shift + 0x8; + // Append byte to current 4-byte chunk 'k' + k = k | ((long) (b & 0xFF) << shift); + shift += 8; - if (shift == 0x20) { - h = 0x00000000FFFFFFFFL & h; + // If chunk is full (32 bits), mix it into 'h' + if (shift == 32) { + h = 0x00000000FFFFFFFFL & h; - k = k * m; - k = 0x00000000FFFFFFFFL & k; + k = k * m; + k = 0x00000000FFFFFFFFL & k; - k = k ^ (k >> r); - k = 0x00000000FFFFFFFFL & k; + k = k ^ (k >> r); + k = 0x00000000FFFFFFFFL & k; - k = k * m; - k = 0x00000000FFFFFFFFL & k; + k = k * m; + k = 0x00000000FFFFFFFFL & k; - h = h * m; - h = 0x00000000FFFFFFFFL & h; + h = h * m; + h = 0x00000000FFFFFFFFL & h; - h = h ^ k; - h = 0x00000000FFFFFFFFL & h; + h = h ^ k; + h = 0x00000000FFFFFFFFL & h; - k = 0x0; - shift = 0x0; + // Reset chunk + k = 0; + shift = 0; + } + } } } diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java b/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java index 931fab77f..999e0b5a7 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java @@ -1,6 +1,6 @@ package pl.skidam.automodpack_core.utils; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.config.Jsons; import java.nio.file.Path; @@ -21,13 +21,13 @@ public Set getWorkaroundMods(Jsons.ModpackContentFields modpackContentFi Set workaroundMods = new HashSet<>(); // this workaround is needed only for neo/forge mods - if (GlobalVariables.LOADER == null || !GlobalVariables.LOADER.contains("forge")) { + if (Constants.LOADER == null || !Constants.LOADER.contains("forge")) { return workaroundMods; } for (Jsons.ModpackContentFields.ModpackContentItem item : modpackContentFields.list) { if (item.type.equals("mod")) { - Path modPath = CustomFileUtils.getPath(modpackPath, item.file); + Path modPath = SmartFileUtils.getPath(modpackPath, item.file); if (FileInspection.hasSpecificServices(modPath)) { workaroundMods.add(item.file); } diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java new file mode 100644 index 000000000..32a9ed4d5 --- /dev/null +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java @@ -0,0 +1,141 @@ +package pl.skidam.automodpack_core.utils.cache; + +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import pl.skidam.automodpack_core.utils.SmartFileUtils; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.concurrent.atomic.AtomicInteger; + +import static pl.skidam.automodpack_core.Constants.LOGGER; + +public class FileMetadataCache implements AutoCloseable { + + private final MVStore store; + private final MVMap fileMetadataMap; + private final AtomicInteger uncommittedWrites = new AtomicInteger(0); + private static final int COMMIT_THRESHOLD = 50; + + private final Object[] locks = new Object[64]; + + public record CachedFile( + String contentHash, + long lastModified, + long size, + String fileKey + ) implements Serializable { + @java.io.Serial + private static final long serialVersionUID = 1L; + } + + public FileMetadataCache(Path dbPath) { + this.store = new MVStore.Builder() + .fileName(dbPath.toString()) + .cacheSize(20) + .autoCommitDisabled() + .open(); + + this.fileMetadataMap = store.openMap("file_metadata"); + + for (int i = 0; i < locks.length; i++) { + locks[i] = new Object(); + } + } + + public String getOrComputeHash(Path file) throws IOException { + Path absPath = file.toAbsolutePath().normalize(); + String pathKey = absPath.toString(); + + BasicFileAttributes attrs = Files.readAttributes(absPath, BasicFileAttributes.class); + long currentSize = attrs.size(); + long currentTime = attrs.lastModifiedTime().toMillis(); + String currentFileKey = attrs.fileKey() != null ? attrs.fileKey().toString() : "null"; + + CachedFile cached = fileMetadataMap.get(pathKey); + if (isCacheValid(cached, currentSize, currentTime, currentFileKey)) { + return cached.contentHash(); // CACHE HIT + } + + // Calculate which lock bucket to use + int lockIndex = Math.abs(pathKey.hashCode() % locks.length); + + synchronized (locks[lockIndex]) { + // Check if another thread has already updated the cache + cached = fileMetadataMap.get(pathKey); + if (isCacheValid(cached, currentSize, currentTime, currentFileKey)) { + return cached.contentHash(); + } + + // Actual work happens here + String newHash = SmartFileUtils.getHash(absPath); + + CachedFile newRecord = new CachedFile(newHash, currentTime, currentSize, currentFileKey); + fileMetadataMap.put(pathKey, newRecord); + checkAndCommit(); + + return newHash; + } + } + + private boolean isCacheValid(CachedFile cached, long size, long time, String key) { + if (cached == null) return false; + return cached.size() == size && + cached.lastModified() == time && + cached.fileKey().equals(key); + } + + public String getHashOrNull(Path path) { + try { + return getOrComputeHash(path); + } catch (IOException e) { + LOGGER.error("Failed to compute hash for path: {}", path, e); + return null; + } + } + + public boolean fastHashCompare(Path file1, Path file2) throws IOException { + if (!Files.exists(file1) || !Files.exists(file2)) return false; + + String hash1 = getOrComputeHash(file1); + String hash2 = getOrComputeHash(file2); + + if (hash1 == null || hash2 == null) return false; + + return hash1.equals(hash2); + } + + // Use only if you are SURE of the file state! + public void overwriteCache(Path file, String hash) throws IOException { + Path absPath = file.toAbsolutePath().normalize(); + String pathKey = absPath.toString(); + + BasicFileAttributes attrs = Files.readAttributes(absPath, BasicFileAttributes.class); + long currentSize = attrs.size(); + long currentTime = attrs.lastModifiedTime().toMillis(); + String currentFileKey = attrs.fileKey() != null ? attrs.fileKey().toString() : "null"; + + CachedFile newRecord = new CachedFile(hash, currentTime, currentSize, currentFileKey); + fileMetadataMap.put(pathKey, newRecord); + checkAndCommit(); + } + + private void checkAndCommit() { + if (uncommittedWrites.incrementAndGet() >= COMMIT_THRESHOLD) { + uncommittedWrites.set(0); + store.commit(); + } + } + + + @Override + public void close() { + if (!store.isClosed()) { + store.commit(); + store.close(); + } + } +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/LauncherVersionSwapper.java b/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/LauncherVersionSwapper.java index a7637073c..0bde471ec 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/LauncherVersionSwapper.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/LauncherVersionSwapper.java @@ -8,7 +8,7 @@ import java.nio.file.Path; import java.util.function.Predicate; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class LauncherVersionSwapper { diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/MultiMCMeta.java b/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/MultiMCMeta.java index 54ed02875..dea6db915 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/MultiMCMeta.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/MultiMCMeta.java @@ -3,7 +3,7 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import java.nio.file.Path; import java.util.Map; @@ -55,18 +55,18 @@ public static boolean updateLoaderVersion(String loaderType, String newVersion) } if (changed) { - var preloadDeltaTime = System.currentTimeMillis() - GlobalVariables.PRELOAD_TIME; + var preloadDeltaTime = System.currentTimeMillis() - Constants.PRELOAD_TIME; var delayRequired = DELAY - preloadDeltaTime; if (delayRequired > 0) { try { // Hack for prism, it reverts our changes if we write them too quickly after launching the game?!? - GlobalVariables.LOGGER.info("Simulating a {} sec delay to avoid MultiMC/Prism overwrite issue...", delayRequired / 1000); + Constants.LOGGER.info("Simulating a {} sec delay to avoid MultiMC/Prism overwrite issue...", delayRequired / 1000); Thread.sleep(delayRequired); } catch (InterruptedException e) { - GlobalVariables.LOGGER.error("Interrupted while simulating delay", e); + Constants.LOGGER.error("Interrupted while simulating delay", e); } } json.add("components", components); - GlobalVariables.LOGGER.info("MultiMC/Prism: Updated loader version to {}", newVersion); + Constants.LOGGER.info("MultiMC/Prism: Updated loader version to {}", newVersion); } return changed; diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/PandoraMeta.java b/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/PandoraMeta.java index 929c707a3..d1ff8190e 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/PandoraMeta.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/launchers/PandoraMeta.java @@ -1,6 +1,6 @@ package pl.skidam.automodpack_core.utils.launchers; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import java.nio.file.Path; @@ -16,7 +16,7 @@ public static boolean updateLoaderVersion(String newVersion) { if (!newVersion.equals(currentVersion)) { json.addProperty("preferred_loader_version", newVersion); - GlobalVariables.LOGGER.info("Pandora: Updated loader version to {}", newVersion); + Constants.LOGGER.info("Pandora: Updated loader version to {}", newVersion); return true; } return false; diff --git a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java index acb1fbbdb..2b3ab7603 100644 --- a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java +++ b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java @@ -3,18 +3,17 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.config.Jsons; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import static org.junit.jupiter.api.Assertions.*; -import static pl.skidam.automodpack_core.GlobalVariables.DEBUG; +import static pl.skidam.automodpack_core.Constants.DEBUG; class ModpackTest { @@ -97,8 +96,8 @@ void modpackTest() { "ModpackContentItems(file=/mods/server-mod-1.20.jar, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)" ); - GlobalVariables.serverConfig = new Jsons.ServerConfigFieldsV2(); - GlobalVariables.serverConfig.autoExcludeUnnecessaryFiles = false; + Constants.serverConfig = new Jsons.ServerConfigFieldsV2(); + Constants.serverConfig.autoExcludeUnnecessaryFiles = false; ModpackContent content = new ModpackContent("TestPack", null, testFilesDir, new HashSet<>(), new HashSet<>(editable), new HashSet<>(), new ModpackExecutor().getExecutor()); content.create(); diff --git a/core/src/test/java/pl/skidam/automodpack_core/utils/CustomFileUtilsTest.java b/core/src/test/java/pl/skidam/automodpack_core/utils/SmartFileUtilsTest.java similarity index 82% rename from core/src/test/java/pl/skidam/automodpack_core/utils/CustomFileUtilsTest.java rename to core/src/test/java/pl/skidam/automodpack_core/utils/SmartFileUtilsTest.java index 74acac1c7..bde3a3d55 100644 --- a/core/src/test/java/pl/skidam/automodpack_core/utils/CustomFileUtilsTest.java +++ b/core/src/test/java/pl/skidam/automodpack_core/utils/SmartFileUtilsTest.java @@ -10,7 +10,7 @@ import static org.junit.jupiter.api.Assertions.*; -class CustomFileUtilsTest { +class SmartFileUtilsTest { @TempDir Path tempDir; @@ -21,7 +21,7 @@ void testSHA1Hash_KnownValue() throws IOException, NoSuchAlgorithmException { String content = "test content 2137!"; Files.writeString(file, content); - String actualHash = CustomFileUtils.getHash(file); + String actualHash = SmartFileUtils.getHash(file); assertNotNull(actualHash, "Hash should not be null"); assertEquals("16883d77e42fcb574c70e31cda49b3f955a48be8", actualHash, "getHash should return the correct SHA-1 hash"); @@ -32,7 +32,7 @@ void testMurmurHash_KnownValue() throws IOException { Path file = tempDir.resolve("murmur-test.txt"); Files.writeString(file, "test content 2137!"); - String actualHash = CustomFileUtils.getCurseforgeMurmurHash(file); + String actualHash = SmartFileUtils.getCurseforgeMurmurHash(file); assertEquals("3151456706", actualHash, "MurmurHash for 'test' should match known constant"); } @@ -48,8 +48,8 @@ void testMurmurHash_IgnoresWhitespace() throws IOException { Files.writeString(cleanFile, cleanContent); Files.writeString(messyFile, messyContent); - String cleanHash = CustomFileUtils.getCurseforgeMurmurHash(cleanFile); - String messyHash = CustomFileUtils.getCurseforgeMurmurHash(messyFile); + String cleanHash = SmartFileUtils.getCurseforgeMurmurHash(cleanFile); + String messyHash = SmartFileUtils.getCurseforgeMurmurHash(messyFile); assertEquals(cleanHash, messyHash, "Hashes should be identical despite whitespace differences"); assertEquals("2667173943", messyHash, "Messy file should still hash to the value of 'test'"); @@ -59,7 +59,7 @@ void testMurmurHash_IgnoresWhitespace() throws IOException { void testGetHash_NonExistentFile() { Path missingFile = tempDir.resolve("does-not-exist.txt"); - String result = CustomFileUtils.getHash(missingFile); + String result = SmartFileUtils.getHash(missingFile); assertNull(result, "Should return null for missing file"); } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Gui.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Gui.java index d8377af32..ede1db9e6 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Gui.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Gui.java @@ -1,6 +1,6 @@ package pl.skidam.automodpack_loader_core; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.utils.PlatformUtils; import javax.imageio.ImageIO; @@ -117,7 +117,7 @@ private void openForked() { if (javaPath == null) throw new RuntimeException("can't find java executable in " + javaBinDir); - Process process = new ProcessBuilder(javaPath.toString(), "-Xmx100M", "-cp", GlobalVariables.THIS_MOD_JAR.toString(), Gui.class.getName(), "--AM.text=" + text) + Process process = new ProcessBuilder(javaPath.toString(), "-Xmx100M", "-cp", Constants.THIS_MOD_JAR.toString(), Gui.class.getName(), "--AM.text=" + text) .redirectOutput(ProcessBuilder.Redirect.INHERIT) .redirectError(ProcessBuilder.Redirect.INHERIT) .start(); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Preload.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Preload.java index 22108f042..5c77d010f 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Preload.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/Preload.java @@ -22,7 +22,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class Preload { @@ -30,7 +30,7 @@ public Preload() { try { long start = System.currentTimeMillis(); LOGGER.info("Prelaunching AutoModpack..."); - initializeGlobalVariables(); + initializeConstants(); loadConfigs(); updateAll(); LOGGER.info("AutoModpack prelaunched! took " + (System.currentTimeMillis() - start) + "ms"); @@ -41,7 +41,6 @@ public Preload() { } private void updateAll() { - var optionalSelectedModpackDir = ModpackContentTools.getModpackDir(clientConfig.selectedModpack); if (LOADER_MANAGER.getEnvironmentType() == LoaderManagerService.EnvironmentType.SERVER || optionalSelectedModpackDir.isEmpty()) { @@ -63,7 +62,7 @@ private void updateAll() { // Only selfupdate if no modpack is selected if (selectedModpackAddress == null) { SelfUpdater.update(); - ClientCacheUtils.deleteDummyFiles(); + LegacyClientCacheUtils.deleteDummyFiles(); } else { Secrets.Secret secret = SecretsStore.getClientSecret(clientConfig.selectedModpack); @@ -82,7 +81,7 @@ private void updateAll() { } // Delete dummy files - ClientCacheUtils.deleteDummyFiles(); + LegacyClientCacheUtils.deleteDummyFiles(); if (clientConfig.updateSelectedModpackOnLaunch) { new ModpackUpdater(latestModpackContent, modpackAddresses, secret, selectedModpackDir).processModpackUpdate(null); @@ -91,7 +90,7 @@ private void updateAll() { } - private void initializeGlobalVariables() { + private void initializeConstants() { // Initialize global variables preload = true; PRELOAD_TIME = System.currentTimeMillis(); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/ReLauncher.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/ReLauncher.java index ded4c0315..2c084081b 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/ReLauncher.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/ReLauncher.java @@ -10,7 +10,7 @@ import java.nio.file.Path; import java.util.concurrent.Semaphore; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class ReLauncher { diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java index 99925921b..a84ffced6 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java @@ -1,7 +1,7 @@ package pl.skidam.automodpack_loader_core; import pl.skidam.automodpack_core.config.Jsons; -import pl.skidam.automodpack_core.utils.CustomFileUtils; +import pl.skidam.automodpack_core.utils.SmartFileUtils; import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.utils.LockFreeInputStream; import pl.skidam.automodpack_core.utils.SemanticVersion; @@ -24,7 +24,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class SelfUpdater { @@ -102,7 +102,7 @@ public static boolean update(Jsons.ModpackContentFields serverModpackContent) { } // Exact Hash Match (Fastest check) - if (automodpack.SHA1Hash().equals(CustomFileUtils.getHash(THIS_MOD_JAR))) { + if (automodpack.SHA1Hash().equals(SmartFileUtils.getHash(THIS_MOD_JAR))) { LOGGER.info("Already on the target version (Hash match): {}", AM_VERSION); return false; } @@ -189,12 +189,12 @@ public static void installModVersion(ModrinthAPI automodpack) { var relauncher = new ReLauncher(updateType); Runnable callback = () -> { - CustomFileUtils.executeOrder66(THIS_MOD_JAR); + SmartFileUtils.executeOrder66(THIS_MOD_JAR); LOGGER.info("Successfully updated AutoModpack! Restarting..."); }; - CustomFileUtils.copyFile(automodpackUpdateJar, newAutomodpackJar); - CustomFileUtils.executeOrder66(automodpackUpdateJar); // Delete temp file + SmartFileUtils.copyFile(automodpackUpdateJar, newAutomodpackJar); + SmartFileUtils.executeOrder66(automodpackUpdateJar); // Delete temp file relauncher.restart(true, callback); } catch (Exception e) { diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index 265de02e9..ae3a15d3b 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -6,6 +6,7 @@ import pl.skidam.automodpack_core.config.ConfigTools; import pl.skidam.automodpack_core.protocol.DownloadClient; import pl.skidam.automodpack_core.utils.*; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import pl.skidam.automodpack_core.utils.launchers.LauncherVersionSwapper; import pl.skidam.automodpack_loader_core.ReLauncher; import pl.skidam.automodpack_loader_core.screen.ScreenManager; @@ -19,7 +20,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import static pl.skidam.automodpack_core.config.ConfigTools.GSON; // TODO: clean up this mess @@ -63,7 +64,9 @@ public void processModpackUpdate(ModpackUtils.UpdateCheckResult result) { // Handle the case where serverModpackContent is null if (serverModpackContent == null) { - CheckAndLoadModpack(); + try (var cache = new FileMetadataCache(hashCacheDBFile)) { + CheckAndLoadModpack(cache); + } return; } @@ -98,7 +101,9 @@ public void processModpackUpdate(ModpackUtils.UpdateCheckResult result) { startUpdate(result.filesToUpdate()); } else { Files.writeString(modpackContentFile, serverModpackContentJson); - CheckAndLoadModpack(); + try (var cache = new FileMetadataCache(hashCacheDBFile)) { + CheckAndLoadModpack(cache); + } } } } catch (Exception e) { @@ -106,15 +111,14 @@ public void processModpackUpdate(ModpackUtils.UpdateCheckResult result) { } } - private void CheckAndLoadModpack() throws Exception { + private void CheckAndLoadModpack(FileMetadataCache cache) throws Exception { if (!Files.exists(modpackDir)) return; - boolean requiresRestart = applyModpack(); + boolean requiresRestart = applyModpack(cache); if (requiresRestart) { LOGGER.info("Modpack is not loaded"); - ClientCacheUtils.saveMetadataCache(); UpdateType updateType = fullDownload ? UpdateType.FULL : UpdateType.UPDATE; new ReLauncher(modpackDir, updateType, changelogs).restart(true); return; @@ -122,30 +126,36 @@ private void CheckAndLoadModpack() throws Exception { // Load the modpack excluding mods from standard mods directory without need to restart the game if (preload) { - List standardModsHashes; + Set standardModsHashes; List modpackMods = List.of(); + // 1. Collect hashes of existing standard mods into a Set for fast lookup try (Stream standardModsStream = Files.list(MODS_DIR)) { standardModsHashes = standardModsStream - .map(ClientCacheUtils::computeHashIfNeeded) + .filter(path -> Files.isRegularFile(path) && path.toString().endsWith(".jar")) // Check extension/type before hashing + .map(cache::getHashOrNull) // Safe wrapper for IOException .filter(Objects::nonNull) - .toList(); + .collect(Collectors.toSet()); // Use Set for O(1) performance + } catch (IOException e) { + LOGGER.error("Failed to list standard mods directory", e); + standardModsHashes = Collections.emptySet(); } + // 2. Filter modpack mods excluding those already present in standard mods Path modpackModsDir = modpackDir.resolve("mods"); if (Files.exists(modpackModsDir)) { try (Stream modpackModsStream = Files.list(modpackModsDir)) { + final Set finalStandardModsHashes = standardModsHashes; modpackMods = modpackModsStream + .filter(path -> Files.isRegularFile(path) && path.toString().endsWith(".jar")) .filter(mod -> { - String modHash = ClientCacheUtils.computeHashIfNeeded(mod); - - // if its in standard mods directory, we don't want to load it again - boolean isUnique = standardModsHashes.stream().noneMatch(hash -> hash.equals(modHash)); - boolean endsWithJar = mod.toString().endsWith(".jar"); - boolean isFile = mod.toFile().isFile(); - - return isUnique && endsWithJar && isFile; - }).toList(); + String modHash = cache.getHashOrNull(mod); + // Only load if hash is valid AND not found in standard set + return modHash != null && !finalStandardModsHashes.contains(modHash); + }) + .toList(); + } catch (IOException e) { + LOGGER.error("Failed to list modpack mods directory", e); } } @@ -153,7 +163,6 @@ private void CheckAndLoadModpack() throws Exception { return; } - ClientCacheUtils.saveMetadataCache(); LOGGER.info("Modpack is already loaded"); } @@ -167,24 +176,23 @@ public void startUpdate(Set files new ScreenManager().download(downloadManager, getModpackName()); long start = System.currentTimeMillis(); - // Don't download files which already exist - // The existing files will be copied over in the applyModpack method - var finalFilesToUpdate = ModpackUtils.getOnlyNonExistingFiles(filesToUpdate); - try { + try (var cache = new FileMetadataCache(hashCacheDBFile)) { + // Don't download files which already exist + // The existing files will be copied over in the applyModpack method + var finalFilesToUpdate = ModpackUtils.getOnlyNonExistingFiles(filesToUpdate, cache); + // Rename modpack modpackDir = ModpackUtils.renameModpackDir(serverModpackContent, modpackDir); modpackContentFile = modpackDir.resolve(modpackContentFile.getFileName()); // FETCH - long startFetching = System.currentTimeMillis(); List fetchDatas = new LinkedList<>(); for (Jsons.ModpackContentFields.ModpackContentItem serverItem : finalFilesToUpdate) { totalBytesToDownload += Long.parseLong(serverItem.size); - String fileType = serverItem.type; // Check if the file is mod, shaderpack or resourcepack is available to download from modrinth or curseforge @@ -200,198 +208,217 @@ public void startUpdate(Set files // DOWNLOAD + try { + downloadModpack(finalFilesToUpdate, startFetching, cache); - newDownloadedFiles.clear(); - int wholeQueue = finalFilesToUpdate.size(); - if (wholeQueue > 0) { - LOGGER.info("In queue left {} files to download ({}MB)", wholeQueue, totalBytesToDownload / 1024 / 1024); + LOGGER.info("Done, saving {}", modpackContentFile); - DownloadClient downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), - Math.min(wholeQueue, 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); - if (downloadClient == null) { - return; - } + // Downloads completed, save json files + Files.writeString(modpackContentFile, serverModpackContentJson); + } catch (Exception e) { + downloadManager.cancelAllAndShutdown(); + LOGGER.error("Error during modpack download", e); + } - downloadManager = new DownloadManager(totalBytesToDownload); - new ScreenManager().download(downloadManager, getModpackName()); - downloadManager.attachDownloadClient(downloadClient); + LegacyClientCacheUtils.deleteDummyFiles(); - var randomizedList = new ArrayList<>(finalFilesToUpdate); - Collections.shuffle(randomizedList); - for (var serverItem : randomizedList) { + if (!failedDownloads.isEmpty()) { + StringBuilder failedFiles = new StringBuilder(); + for (var download : failedDownloads.entrySet()) { + var item = download.getKey(); + var urls = download.getValue(); + LOGGER.error("{}{}", "Failed to download: " + item.file + " from ", urls); + failedFiles.append(item.file); + } - String serverFilePath = serverItem.file; - String serverHash = serverItem.sha1; + new ScreenManager().error("automodpack.error.files", "Failed to download: " + failedFiles, "automodpack.error.logs"); + LOGGER.error("Update failed successfully! Try again! Took: {}ms", System.currentTimeMillis() - start); + } else if (preload) { + LOGGER.info("Update completed! Took: {}ms", System.currentTimeMillis() - start); + CheckAndLoadModpack(cache); + } else { + boolean requiredRestart = applyModpack(cache); + LOGGER.info("Update completed! Required restart: {} Took: {}ms", requiredRestart, System.currentTimeMillis() - start); + UpdateType updateType = fullDownload ? UpdateType.FULL : UpdateType.UPDATE; + new ReLauncher(modpackDir, updateType, changelogs).restart(false); + } + } catch (SocketTimeoutException | ConnectException e) { + LOGGER.error("{} is not responding", "Modpack host of " + modpackAddresses.hostAddress, e); + } catch (InterruptedException e) { + LOGGER.info("Interrupted the download"); + } catch (Exception e) { + new ScreenManager().error("automodpack.error.critical", "\"" + e.getMessage() + "\"", "automodpack.error.logs"); + LOGGER.error("Critical error during modpack update", e); + } + } - Path downloadFile = CustomFileUtils.getPath(modpackDir, serverFilePath); + private void downloadModpack(Set finalFilesToUpdate, long startFetching, FileMetadataCache cache) throws InterruptedException { + int wholeQueue = finalFilesToUpdate.size(); - if (!Files.exists(downloadFile)) { - newDownloadedFiles.add(serverFilePath); - } + if (wholeQueue > 0) { + LOGGER.info("In queue left {} files to download ({}MB)", wholeQueue, totalBytesToDownload / 1024 / 1024); - List urls = new ArrayList<>(); - if (fetchManager.getFetchDatas().containsKey(serverHash)) { - urls.addAll(fetchManager.getFetchDatas().get(serverHash).fetchedData().urls()); - } + DownloadClient downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), + Math.min(wholeQueue, 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); + if (downloadClient == null) { + return; + } - Runnable failureCallback = () -> { - failedDownloads.put(serverItem, urls); - }; + downloadManager = new DownloadManager(totalBytesToDownload); + new ScreenManager().download(downloadManager, getModpackName()); + downloadManager.attachDownloadClient(downloadClient); - Runnable successCallback = () -> { - List mainPageUrls = new LinkedList<>(); - if (fetchManager != null && fetchManager.getFetchDatas().get(serverHash) != null) { - mainPageUrls = fetchManager.getFetchDatas().get(serverHash).fetchedData().mainPageUrls(); - } + var randomizedList = new ArrayList<>(finalFilesToUpdate); + Collections.shuffle(randomizedList); + for (var serverItem : randomizedList) { - changelogs.changesAddedList.put(downloadFile.getFileName().toString(), mainPageUrls); + String serverFilePath = serverItem.file; + String serverHash = serverItem.sha1; - ClientCacheUtils.updateCache(downloadFile, serverHash); - }; + Path downloadFile = SmartFileUtils.getPath(modpackDir, serverFilePath); + if (!Files.exists(downloadFile)) { + newDownloadedFiles.add(serverFilePath); + } - downloadManager.download(downloadFile, serverHash, urls, successCallback, failureCallback); + List urls = new ArrayList<>(); + if (fetchManager.getFetchDatas().containsKey(serverHash)) { + urls.addAll(fetchManager.getFetchDatas().get(serverHash).fetchedData().urls()); } - downloadManager.joinAll(); + Runnable failureCallback = () -> { + failedDownloads.put(serverItem, urls); + }; - LOGGER.info("Finished downloading files in {}ms", System.currentTimeMillis() - startFetching); + Runnable successCallback = () -> { + List mainPageUrls = new LinkedList<>(); + if (fetchManager != null && fetchManager.getFetchDatas().get(serverHash) != null) { + mainPageUrls = fetchManager.getFetchDatas().get(serverHash).fetchedData().mainPageUrls(); + } - if (downloadManager.isCanceled()) { - LOGGER.warn("Download canceled"); - return; - } + changelogs.changesAddedList.put(downloadFile.getFileName().toString(), mainPageUrls); - downloadManager.cancelAllAndShutdown(); - totalBytesToDownload = 0; - - Map hashesToRefresh = new HashMap<>(); // File name, hash - var failedDownloadsSecMap = new HashMap<>(failedDownloads); - failedDownloadsSecMap.forEach((k, v) -> { - hashesToRefresh.put(k.file, k.sha1); - failedDownloads.remove(k); - totalBytesToDownload += Long.parseLong(k.size); - }); + try { + cache.overwriteCache(downloadFile, serverHash); + } catch (Exception e) { + LOGGER.error("Failed to update cache for {}", downloadFile, e); + } + }; - if (!hashesToRefresh.isEmpty()) { - LOGGER.warn("Failed to download {} files", hashesToRefresh.size()); - } - if (!hashesToRefresh.isEmpty()) { - // make byte[][] from hashesToRefresh.values() - byte[][] hashesArray = hashesToRefresh.values().stream() - .map(String::getBytes) - .toArray(byte[][]::new); + downloadManager.download(downloadFile, serverHash, urls, successCallback, failureCallback); + } - // send it to the server and get the new modpack content - // TODO set client to a waiting for the server to respond screen - LOGGER.warn("Trying to refresh the modpack content"); - LOGGER.info("Sending hashes to refresh: {}", hashesToRefresh.values()); - var refreshedContentOptional = ModpackUtils.refreshServerModpackContent(modpackAddresses, modpackSecret, hashesArray, false); - if (refreshedContentOptional.isEmpty()) { - LOGGER.error("Failed to refresh the modpack content"); - } else { - LOGGER.info("Successfully refreshed the modpack content"); - // retry the download - // success - // or fail and then show the error + downloadManager.joinAll(); - var refreshedContent = refreshedContentOptional.get(); - this.serverModpackContent = refreshedContent; - this.serverModpackContentJson = GSON.toJson(refreshedContent); + LOGGER.info("Finished downloading files in {}ms", System.currentTimeMillis() - startFetching); + + if (downloadManager.isCanceled()) { + LOGGER.warn("Download canceled"); + return; + } - // filter list to only the failed downloads - var refreshedFilteredList = refreshedContent.list.stream().filter(item -> hashesToRefresh.containsKey(item.file)).toList(); + downloadManager.cancelAllAndShutdown(); + totalBytesToDownload = 0; - downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), Math.min(refreshedFilteredList.size(), 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); - if (downloadClient == null) { - return; - } - downloadManager = new DownloadManager(totalBytesToDownload); - new ScreenManager().download(downloadManager, getModpackName()); - downloadManager.attachDownloadClient(downloadClient); + Map hashesToRefresh = new HashMap<>(); // File name, hash + var failedDownloadsSecMap = new HashMap<>(failedDownloads); + failedDownloadsSecMap.forEach((k, v) -> { + hashesToRefresh.put(k.file, k.sha1); + failedDownloads.remove(k); + totalBytesToDownload += Long.parseLong(k.size); + }); - // TODO try to fetch again from modrinth and curseforge + if (!hashesToRefresh.isEmpty()) { + LOGGER.warn("Failed to download {} files", hashesToRefresh.size()); + } - randomizedList = new ArrayList<>(refreshedFilteredList); - Collections.shuffle(randomizedList); - for (var serverItem : randomizedList) { + if (!hashesToRefresh.isEmpty()) { + // make byte[][] from hashesToRefresh.values() + byte[][] hashesArray = hashesToRefresh.values().stream() + .map(String::getBytes) + .toArray(byte[][]::new); + + // send it to the server and get the new modpack content + // TODO set client to a waiting for the server to respond screen + LOGGER.warn("Trying to refresh the modpack content"); + LOGGER.info("Sending hashes to refresh: {}", hashesToRefresh.values()); + var refreshedContentOptional = ModpackUtils.refreshServerModpackContent(modpackAddresses, modpackSecret, hashesArray, false); + if (refreshedContentOptional.isEmpty()) { + LOGGER.error("Failed to refresh the modpack content"); + } else { + LOGGER.info("Successfully refreshed the modpack content"); + // retry the download + // success + // or fail and then show the error + + var refreshedContent = refreshedContentOptional.get(); + this.serverModpackContent = refreshedContent; + this.serverModpackContentJson = GSON.toJson(refreshedContent); + + // filter list to only the failed downloads + var refreshedFilteredList = refreshedContent.list.stream().filter(item -> hashesToRefresh.containsKey(item.file)).toList(); + if (refreshedFilteredList.isEmpty()) { + return; + } - String serverFilePath = serverItem.file; - String serverHash = serverItem.sha1; + downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), Math.min(refreshedFilteredList.size(), 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); + if (downloadClient == null) { + return; + } - Path downloadFile = CustomFileUtils.getPath(modpackDir, serverFilePath); + downloadManager = new DownloadManager(totalBytesToDownload); + new ScreenManager().download(downloadManager, getModpackName()); + downloadManager.attachDownloadClient(downloadClient); - LOGGER.info("Retrying to download {} from {}", serverFilePath, modpackAddresses.hostAddress.getHostName()); + // TODO try to fetch again from modrinth and curseforge - Runnable failureCallback = () -> { - failedDownloads.put(serverItem, List.of()); - }; + randomizedList = new ArrayList<>(refreshedFilteredList); + Collections.shuffle(randomizedList); + for (var serverItem : randomizedList) { - Runnable successCallback = () -> { - changelogs.changesAddedList.put(downloadFile.getFileName().toString(), null); + String serverFilePath = serverItem.file; + String serverHash = serverItem.sha1; - ClientCacheUtils.updateCache(downloadFile, serverHash); - }; + Path downloadFile = SmartFileUtils.getPath(modpackDir, serverFilePath); - downloadManager.download(downloadFile, serverHash, List.of(), successCallback, failureCallback); - } + LOGGER.info("Retrying to download {} from {}", serverFilePath, modpackAddresses.hostAddress.getHostName()); - downloadManager.joinAll(); + Runnable failureCallback = () -> { + failedDownloads.put(serverItem, List.of()); + }; - if (downloadManager.isCanceled()) { - LOGGER.warn("Download canceled"); - return; - } + Runnable successCallback = () -> { + changelogs.changesAddedList.put(downloadFile.getFileName().toString(), null); - downloadManager.cancelAllAndShutdown(); + try { + cache.overwriteCache(downloadFile, serverHash); + } catch (Exception e) { + LOGGER.error("Failed to update cache for {}", downloadFile, e); + } + }; - LOGGER.info("Finished refreshed downloading files in {}ms", System.currentTimeMillis() - startFetching); + downloadManager.download(downloadFile, serverHash, List.of(), successCallback, failureCallback); } - } - } - LOGGER.info("Done, saving {}", modpackContentFile); + downloadManager.joinAll(); - // Downloads completed, save json files - Files.writeString(modpackContentFile, serverModpackContentJson); + if (downloadManager.isCanceled()) { + LOGGER.warn("Download canceled"); + return; + } - ClientCacheUtils.saveMetadataCache(); - ClientCacheUtils.deleteDummyFiles(); + downloadManager.cancelAllAndShutdown(); - if (!failedDownloads.isEmpty()) { - StringBuilder failedFiles = new StringBuilder(); - for (var download : failedDownloads.entrySet()) { - var item = download.getKey(); - var urls = download.getValue(); - LOGGER.error("{}{}", "Failed to download: " + item.file + " from ", urls); - failedFiles.append(item.file); + LOGGER.info("Finished refreshed downloading files in {}ms", System.currentTimeMillis() - startFetching); } - - new ScreenManager().error("automodpack.error.files", "Failed to download: " + failedFiles, "automodpack.error.logs"); - LOGGER.error("Update failed successfully! Try again! Took: {}ms", System.currentTimeMillis() - start); - } else if (preload) { - LOGGER.info("Update completed! Took: {}ms", System.currentTimeMillis() - start); - CheckAndLoadModpack(); - } else { - boolean requiredRestart = applyModpack(); - LOGGER.info("Update completed! Required restart: {} Took: {}ms", requiredRestart, System.currentTimeMillis() - start); - UpdateType updateType = fullDownload ? UpdateType.FULL : UpdateType.UPDATE; - new ReLauncher(modpackDir, updateType, changelogs).restart(false); } - } catch (SocketTimeoutException | ConnectException e) { - LOGGER.error("{} is not responding", "Modpack host of " + modpackAddresses.hostAddress, e); - } catch (InterruptedException e) { - LOGGER.info("Interrupted the download"); - } catch (Exception e) { - new ScreenManager().error("automodpack.error.critical", "\"" + e.getMessage() + "\"", "automodpack.error.logs"); - e.printStackTrace(); } } // this is run every time we modpack is updated // returns true if restart is required - private boolean applyModpack() throws Exception { + private boolean applyModpack(FileMetadataCache cache) throws Exception { ModpackUtils.selectModpack(modpackDir, modpackAddresses, newDownloadedFiles); try { // try catch this error there because we don't want to stop the whole method just because of that SecretsStore.saveClientSecret(clientConfig.selectedModpack, modpackSecret); @@ -405,14 +432,14 @@ private boolean applyModpack() throws Exception { } // Prepare modpack, analyze nested mods - List conflictingNestedMods = MODPACK_LOADER.getModpackNestedConflicts(modpackDir); + List conflictingNestedMods = MODPACK_LOADER.getModpackNestedConflicts(modpackDir, cache); // delete old deleted files from the server modpack - boolean needsRestart0 = deleteNonModpackFiles(modpackContent); + boolean needsRestart0 = deleteNonModpackFiles(modpackContent, cache); Set workaroundMods = new WorkaroundUtil(modpackDir).getWorkaroundMods(modpackContent); Set filesNotToCopy = getFilesNotToCopy(modpackContent.list, workaroundMods); - boolean needsRestart1 = ModpackUtils.correctFilesLocations(modpackDir, modpackContent, filesNotToCopy); + boolean needsRestart1 = ModpackUtils.correctFilesLocations(modpackDir, modpackContent, filesNotToCopy, cache); workaroundMods = new WorkaroundUtil(modpackDir).getWorkaroundMods(modpackContent); filesNotToCopy = getFilesNotToCopy(modpackContent.list, workaroundMods); @@ -453,7 +480,7 @@ private boolean applyModpack() throws Exception { LOGGER.warn("Found conflicting nested mods: {}", conflictingNestedMods); } - boolean needsRestart2 = ModpackUtils.fixNestedMods(conflictingNestedMods, standardModList); + boolean needsRestart2 = ModpackUtils.fixNestedMods(conflictingNestedMods, standardModList, cache); Set ignoredFiles = ModpackUtils.getIgnoredFiles(conflictingNestedMods, workaroundMods); Set forceCopyFiles = modpackContent.list.stream() @@ -466,9 +493,9 @@ private boolean applyModpack() throws Exception { boolean needsRestart3 = removeDupeModsResult.requiresRestart(); // Remove rest of mods not for standard mods directory - boolean needsRestart4 = ModpackUtils.removeRestModsNotToCopy(modpackContent, filesNotToCopy, removeDupeModsResult.modsToKeep()); + boolean needsRestart4 = ModpackUtils.removeRestModsNotToCopy(modpackContent, filesNotToCopy, removeDupeModsResult.modsToKeep(), cache); - boolean needsRestart5 = ModpackUtils.deleteFilesMarkedForDeletionByTheServer(modpackContent.nonModpackFilesToDelete); + boolean needsRestart5 = ModpackUtils.deleteFilesMarkedForDeletionByTheServer(modpackContent.nonModpackFilesToDelete, cache); boolean needsRestart6 = LauncherVersionSwapper.swapLoaderVersion(modpackContent.loader, modpackContent.loaderVersion); @@ -500,7 +527,7 @@ private Set getFilesNotToCopy(Set modpackFiles = modpackContent.list.stream().map(modpackContentField -> modpackContentField.file).collect(Collectors.toSet()); List pathList; try (Stream pathStream = Files.walk(modpackDir)) { @@ -514,27 +541,27 @@ private boolean deleteNonModpackFiles(Jsons.ModpackContentFields modpackContent) continue; } - String formattedFile = CustomFileUtils.formatPath(path, modpackDir); + String formattedFile = SmartFileUtils.formatPath(path, modpackDir); if (modpackFiles.contains(formattedFile)) { continue; } - Path runPath = CustomFileUtils.getPathFromCWD(formattedFile); - if (ClientCacheUtils.fastHashCompare(path, runPath)) { + Path runPath = SmartFileUtils.getPathFromCWD(formattedFile); + if (cache.fastHashCompare(path, runPath)) { LOGGER.info("Deleting {} and {}", path, runPath); parentPaths.add(runPath.getParent()); - CustomFileUtils.executeOrder66(runPath, false); + SmartFileUtils.executeOrder66(runPath, false); needsRestart = true; } else { LOGGER.info("Deleting {}", path); } parentPaths.add(path.getParent()); - CustomFileUtils.executeOrder66(path, false); + SmartFileUtils.executeOrder66(path, false); changelogs.changesDeletedList.put(path.getFileName().toString(), null); } - ClientCacheUtils.saveDummyFiles(); + LegacyClientCacheUtils.saveDummyFiles(); // recursively delete empty directories for (Path parentPath : parentPaths) { @@ -545,12 +572,12 @@ private boolean deleteNonModpackFiles(Jsons.ModpackContentFields modpackContent) } private void deleteEmptyParentDirectoriesRecursively(Path directory) throws IOException { - if (directory == null || !CustomFileUtils.isEmptyDirectory(directory)) { + if (directory == null || !SmartFileUtils.isEmptyDirectory(directory)) { return; } LOGGER.info("Deleting empty directory {}", directory); - CustomFileUtils.executeOrder66(directory); + SmartFileUtils.executeOrder66(directory); deleteEmptyParentDirectoriesRecursively(directory.getParent()); } } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java index 9d0a43b0d..cdb6ac24a 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java @@ -5,10 +5,11 @@ import pl.skidam.automodpack_core.config.Jsons; import pl.skidam.automodpack_core.protocol.DownloadClient; import pl.skidam.automodpack_core.protocol.NetUtils; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; -import pl.skidam.automodpack_core.utils.CustomFileUtils; +import pl.skidam.automodpack_core.utils.LegacyClientCacheUtils; +import pl.skidam.automodpack_core.utils.SmartFileUtils; import pl.skidam.automodpack_core.utils.FileInspection; import pl.skidam.automodpack_core.utils.ModpackContentTools; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import pl.skidam.automodpack_loader_core.screen.ScreenManager; import java.io.*; @@ -24,8 +25,8 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static pl.skidam.automodpack_core.GlobalVariables.*; -import static pl.skidam.automodpack_core.utils.ClientCacheUtils.*; +import static pl.skidam.automodpack_core.Constants.*; +import static pl.skidam.automodpack_core.utils.LegacyClientCacheUtils.*; public class ModpackUtils { @@ -63,33 +64,26 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac Set filesToUpdate = ConcurrentHashMap.newKeySet(); - serverModpackContent.list.forEach(serverItem -> { - Path serverItemPath = CustomFileUtils.getPath(modpackDir, serverItem.file); - if (!existingFileTree.contains(serverItemPath)) { - filesToUpdate.add(serverItem); // File is missing - return; - } else if (serverItem.editable) { // TODO check if this is enough of a check, what if user already had a file but there's provided the same by a new modpack version which wasnt in the modpack before? - LOGGER.debug("Skipping editable file hash check: {}", serverItem.file); - return; - } - - String cachedHash = ClientCacheUtils.getVerifiedCacheHash(serverItemPath); - if (cachedHash != null && cachedHash.equals(serverItem.sha1)) { - return; // File is almost certainly up to date - } - - // Full hash check - String diskHash = CustomFileUtils.getHash(serverItemPath); - if (diskHash != null && diskHash.equals(serverItem.sha1)) { - ClientCacheUtils.updateCache(serverItemPath, diskHash); - return; // File is definitely up to date - } + try (var cache = new FileMetadataCache(hashCacheDBFile)) { + serverModpackContent.list.forEach(serverItem -> { + Path serverItemPath = SmartFileUtils.getPath(modpackDir, serverItem.file); + if (!existingFileTree.contains(serverItemPath)) { + filesToUpdate.add(serverItem); // File is missing + return; + } else if (serverItem.editable) { // TODO check if this is enough of a check, what if user already had a file but there's provided the same by a new modpack version which wasnt in the modpack before? + LOGGER.debug("Skipping editable file hash check: {}", serverItem.file); + return; + } - // This file needs to be updated - filesToUpdate.add(serverItem); - }); + String hash = cache.getHashOrNull(serverItemPath); + if (hash != null && hash.equals(serverItem.sha1)) { + return; // File is up to date + } - ClientCacheUtils.saveMetadataCache(); + // This file needs to be updated + filesToUpdate.add(serverItem); + }); + } if (!filesToUpdate.isEmpty()) { LOGGER.info("Modpack {} requires update! Took {} ms", modpackDir, System.currentTimeMillis() - start); @@ -116,7 +110,7 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac // TODO check more dirs, preferably with a database of all indexed files by their hashes // This will scan the CWD with CustomFileUtils.getPathFromCWD(modpack.file) and if the file exist with matching hash, it won't be returned in the final set // Used to avoid downloading files that are already present and valid on disk so we can just copy them over instead of downloading them all again - public static Set getOnlyNonExistingFiles(Set filesToCheck) { + public static Set getOnlyNonExistingFiles(Set filesToCheck, FileMetadataCache cache) { Set nonExistingFiles = new HashSet<>(); LOGGER.info("Checking for existing files to skip downloading..."); @@ -124,9 +118,9 @@ public static Set getOnlyNonExist int filesSkipped = 0; for (var entry : filesToCheck) { - Path fileInCWD = CustomFileUtils.getPathFromCWD(entry.file); + Path fileInCWD = SmartFileUtils.getPathFromCWD(entry.file); if (Files.isRegularFile(fileInCWD)) { - String diskHash = ClientCacheUtils.computeHashIfNeeded(fileInCWD); + String diskHash = cache.getHashOrNull(fileInCWD); if (diskHash.equalsIgnoreCase(entry.sha1)) { LOGGER.debug("File already exists and matches hash, skipping download: {}", entry.file); filesSkipped++; @@ -137,14 +131,13 @@ public static Set getOnlyNonExist nonExistingFiles.add(entry); } - ClientCacheUtils.saveMetadataCache(); LOGGER.info("Finished checking for existing files in CWD, {} files left to download (skipped {} existing). Took {} ms", nonExistingFiles.size(), filesSkipped, System.currentTimeMillis() - time); return nonExistingFiles; } - public static boolean deleteFilesMarkedForDeletionByTheServer(Set filesToDeleteOnClient) { + public static boolean deleteFilesMarkedForDeletionByTheServer(Set filesToDeleteOnClient, FileMetadataCache cache) { if (!clientConfig.allowRemoteNonModpackDeletions) { if (!filesToDeleteOnClient.isEmpty()) { LOGGER.warn("Server requested deletion of {} files, but remote deletions are disabled in client config! Consider deleting them manually.", filesToDeleteOnClient.size()); @@ -167,14 +160,14 @@ public static boolean deleteFilesMarkedForDeletionByTheServer(Set { if (!Files.isRegularFile(path)) return; - String diskHash = ClientCacheUtils.computeHashIfNeeded(path); + String diskHash = cache.getHashOrNull(path); if (diskHash.equalsIgnoreCase(expectedHash)) { boolean isModFile = FileInspection.isMod(path); LOGGER.warn("Deleting file marked for deletion by server: {}", path); - CustomFileUtils.executeOrder66(path); + SmartFileUtils.executeOrder66(path); if (isModFile) { deletedAnyModFile.set(true); } @@ -217,28 +210,28 @@ public static boolean deleteFilesMarkedForDeletionByTheServer(Set filesNotToCopy) throws IOException { + public static boolean correctFilesLocations(Path modpackDir, Jsons.ModpackContentFields serverModpackContent, Set filesNotToCopy, FileMetadataCache cache) throws IOException { boolean needsRestart = false; // correct the files locations for (Jsons.ModpackContentFields.ModpackContentItem contentItem : serverModpackContent.list) { String formattedFile = contentItem.file; - Path modpackFile = CustomFileUtils.getPath(modpackDir, formattedFile); - Path runFile = CustomFileUtils.getPathFromCWD(formattedFile); + Path modpackFile = SmartFileUtils.getPath(modpackDir, formattedFile); + Path runFile = SmartFileUtils.getPathFromCWD(formattedFile); boolean isMod = "mod".equals(contentItem.type); if (isMod) { // Make it into standardized mods directory, for support custom launchers - runFile = CustomFileUtils.getPath(MODS_DIR, formattedFile.replaceFirst("/mods/", "")); + runFile = SmartFileUtils.getPath(MODS_DIR, formattedFile.replaceFirst("/mods/", "")); } boolean modpackFileExists = Files.exists(modpackFile); boolean runFileExists = Files.exists(runFile); boolean runFileHashMatch = false; - if (runFileExists) runFileHashMatch = Objects.equals(contentItem.sha1, ClientCacheUtils.computeHashIfNeeded(runFile)); + if (runFileExists) runFileHashMatch = Objects.equals(contentItem.sha1, cache.getHashOrNull(runFile)); if (runFileHashMatch && !modpackFileExists) { LOGGER.debug("Copying {} file to the modpack directory", formattedFile); - CustomFileUtils.copyFile(runFile, modpackFile); + SmartFileUtils.copyFile(runFile, modpackFile); modpackFileExists = true; } @@ -249,7 +242,7 @@ public static boolean correctFilesLocations(Path modpackDir, Jsons.ModpackConten } if (modpackFileExists && !runFileExists) { - CustomFileUtils.copyFile(modpackFile, runFile); + SmartFileUtils.copyFile(modpackFile, runFile); if (isMod) { needsRestart = true; @@ -259,7 +252,7 @@ public static boolean correctFilesLocations(Path modpackDir, Jsons.ModpackConten LOGGER.error("File {} doesn't exist!? If you see this please report this to the automodpack repo and attach this log https://github.com/Skidamek/AutoModpack/issues", formattedFile); Thread.dumpStack(); } else if (!runFileHashMatch) { - CustomFileUtils.copyFile(modpackFile, runFile); + SmartFileUtils.copyFile(modpackFile, runFile); if (isMod) { needsRestart = true; LOGGER.warn("Overwriting mod {} file to modpack version", formattedFile); @@ -272,16 +265,16 @@ public static boolean correctFilesLocations(Path modpackDir, Jsons.ModpackConten return needsRestart; } - public static boolean removeRestModsNotToCopy(Jsons.ModpackContentFields serverModpackContent, Set filesNotToCopy, Set modsToKeep) { + public static boolean removeRestModsNotToCopy(Jsons.ModpackContentFields serverModpackContent, Set filesNotToCopy, Set modsToKeep, FileMetadataCache cache) { boolean needsRestart = false; for (Jsons.ModpackContentFields.ModpackContentItem contentItem : serverModpackContent.list) { String formattedFile = contentItem.file; - Path runFile = CustomFileUtils.getPathFromCWD(formattedFile); + Path runFile = SmartFileUtils.getPathFromCWD(formattedFile); boolean isMod = "mod".equals(contentItem.type); if (isMod) { // Make it into standardized mods directory, for support custom launchers - runFile = CustomFileUtils.getPath(MODS_DIR, formattedFile.replaceFirst("/mods/", "")); + runFile = SmartFileUtils.getPath(MODS_DIR, formattedFile.replaceFirst("/mods/", "")); } if (modsToKeep.contains(runFile)) { @@ -291,11 +284,11 @@ public static boolean removeRestModsNotToCopy(Jsons.ModpackContentFields serverM boolean runFileExists = Files.exists(runFile); boolean runFileHashMatch = false; - if (runFileExists) runFileHashMatch = Objects.equals(contentItem.sha1, ClientCacheUtils.computeHashIfNeeded(runFile)); + if (runFileExists) runFileHashMatch = contentItem.sha1.equalsIgnoreCase(cache.getHashOrNull(runFile)); if (runFileHashMatch && isMod && filesNotToCopy.contains(formattedFile)) { LOGGER.info("Deleting {} file from standard mods directory", formattedFile); - CustomFileUtils.executeOrder66(runFile); + SmartFileUtils.executeOrder66(runFile); needsRestart = true; } } @@ -305,7 +298,7 @@ public static boolean removeRestModsNotToCopy(Jsons.ModpackContentFields serverM // Copies necessary nested mods from modpack mods to standard mods folder // Returns true if requires client restart - public static boolean fixNestedMods(List conflictingNestedMods, Collection standardModList) throws IOException { + public static boolean fixNestedMods(List conflictingNestedMods, Collection standardModList, FileMetadataCache cache) throws IOException { if (conflictingNestedMods.isEmpty()) return false; @@ -313,16 +306,16 @@ public static boolean fixNestedMods(List conflictingNestedMo boolean needsRestart = false; for (FileInspection.Mod mod : conflictingNestedMods) { - // Check mods provides, if theres some mod which is named with the same id as some other mod 'provides' remove the mod which provides that id as well, otherwise loader will crash + // Check mods provides, if there's some mod which is named with the same id as some other mod 'provides' remove the mod which provides that id as well, otherwise loader will crash if (standardModIDs.stream().anyMatch(mod.providesIDs()::contains)) continue; Path modPath = mod.modPath(); Path standardModPath = MODS_DIR.resolve(modPath.getFileName()); - if (!Files.exists(standardModPath) || !Objects.equals(ClientCacheUtils.computeHashIfNeeded(standardModPath), mod.hash())) { + if (!Files.exists(standardModPath) || !mod.hash().equalsIgnoreCase(cache.getHashOrNull(standardModPath))) { needsRestart = true; LOGGER.info("Copying nested mod {} to standard mods folder", standardModPath.getFileName()); - CustomFileUtils.copyFile(modPath, standardModPath); + SmartFileUtils.copyFile(modPath, standardModPath); var newMod = FileInspection.getMod(standardModPath); if (newMod != null) standardModList.add(newMod); // important } @@ -336,7 +329,7 @@ public static Set getIgnoredFiles(List conflictingNe Set newIgnoredFiles = new HashSet<>(workarounds); for (FileInspection.Mod mod : conflictingNestedMods) { - newIgnoredFiles.add(CustomFileUtils.formatPath(mod.modPath(), modpacksDir)); + newIgnoredFiles.add(SmartFileUtils.formatPath(mod.modPath(), modpacksDir)); } return newIgnoredFiles; @@ -350,7 +343,7 @@ public static Map getDupeMods(Path modpa for (FileInspection.Mod modpackMod : modpackModList) { FileInspection.Mod standardMod = standardModList.stream().filter(mod -> mod.modID().equals(modpackMod.modID())).findFirst().orElse(null); // There might be super rare edge case if client would have for some reason more than one mod with the same mod id if (standardMod != null) { - String formattedFile = CustomFileUtils.formatPath(modpackMod.modPath(), modpackDir); + String formattedFile = SmartFileUtils.formatPath(modpackMod.modPath(), modpackDir); if (ignoredMods.contains(formattedFile) || forceCopyFiles.contains(formattedFile)) continue; @@ -400,7 +393,7 @@ public static RemoveDupeModsResult removeDupeMods(Path modpackDir, Collection providesIDs = modpackMod.providesIDs(); List IDs = new ArrayList<>(providesIDs); IDs.add(modId); @@ -417,18 +410,18 @@ public static RemoveDupeModsResult removeDupeMods(Path modpackDir, Collection { + if (isHashInvalid(item.sha1)) { + LOGGER.error("Modpack content is invalid: file '{}' has invalid sha1 '{}'", item.file, item.sha1); + return true; + } if (isUnsafePath(item.file, false)) { LOGGER.error("Modpack content is invalid: file path '{}' is unsafe/malicious", item.file); return true; @@ -700,6 +697,10 @@ public static boolean potentiallyMalicious(Jsons.ModpackContentFields serverModp }); boolean nonModpackFilesToDeleteInvalid = serverModpackContent.nonModpackFilesToDelete.stream().anyMatch(item -> { + if (isHashInvalid(item.sha1)) { + LOGGER.error("Modpack content is invalid: file '{}' has invalid sha1 '{}'", item.file, item.sha1); + return true; + } if (isUnsafePath(item.file, false)) { LOGGER.error("Modpack content is invalid: file to delete path '{}' is unsafe/malicious", item.file); return true; @@ -710,6 +711,16 @@ public static boolean potentiallyMalicious(Jsons.ModpackContentFields serverModp return listInvalid || nonModpackFilesToDeleteInvalid; } + // Assumes sha1 hash + private static boolean isHashInvalid(String hash) { + if (hash == null || hash.isBlank()) { + return true; + } + + // SHA-1 hashes are 40 hexadecimal characters + return !hash.matches("^[a-fA-F0-9]{40}$"); + } + private static boolean isUnsafePath(String rawPath, boolean blankIsFine) { if (rawPath == null) return true; @@ -748,10 +759,10 @@ public static void preserveEditableFiles(Path modpackDir, Set editableFi // Here, mods can be copied, no problem - Path path = CustomFileUtils.getPathFromCWD(file); + Path path = SmartFileUtils.getPathFromCWD(file); if (Files.exists(path)) { try { - CustomFileUtils.copyFile(path, CustomFileUtils.getPath(modpackDir, file)); + SmartFileUtils.copyFile(path, SmartFileUtils.getPath(modpackDir, file)); } catch (IOException e) { e.printStackTrace(); } @@ -767,10 +778,10 @@ public static void copyPreviousEditableFiles(Path modpackDir, Set editab if (file.contains("/mods/") && file.endsWith(".jar")) // Don't mess with mods here, it will cause issues continue; - Path path = CustomFileUtils.getPath(modpackDir, file); + Path path = SmartFileUtils.getPath(modpackDir, file); if (Files.exists(path)) { try { - CustomFileUtils.copyFile(path, CustomFileUtils.getPathFromCWD(file)); + SmartFileUtils.copyFile(path, SmartFileUtils.getPathFromCWD(file)); } catch (IOException e) { e.printStackTrace(); } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/compat/crashassistant/ProcessSignalIO.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/compat/crashassistant/ProcessSignalIO.java index f8d185af1..2410be1a6 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/compat/crashassistant/ProcessSignalIO.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/compat/crashassistant/ProcessSignalIO.java @@ -1,6 +1,6 @@ package pl.skidam.automodpack_loader_core.compat.crashassistant; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -56,7 +56,7 @@ public static void post(String name, String data) { StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); } catch (IOException e) { - GlobalVariables.LOGGER.error("Error while saving data to {}", fileName, e); + Constants.LOGGER.error("Error while saving data to {}", fileName, e); } } @@ -88,7 +88,7 @@ public static void postAsOtherProcess(String name, String data, long pid) { StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); } catch (IOException e) { - GlobalVariables.LOGGER.error("Error while saving data to {}", fileName, e); + Constants.LOGGER.error("Error while saving data to {}", fileName, e); } } @@ -115,7 +115,7 @@ public static Optional get(String name, long pid) { String content = new String(bytes, StandardCharsets.UTF_8); return Optional.of(content); } catch (IOException e) { - GlobalVariables.LOGGER.error("Error while reading data from {}", fileName, e); + Constants.LOGGER.error("Error while reading data from {}", fileName, e); return Optional.empty(); } } @@ -153,7 +153,7 @@ public static void postInfo(String name, String data) { StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); } catch (IOException e) { - GlobalVariables.LOGGER.error("Error while saving info to {}", fileName, e); + Constants.LOGGER.error("Error while saving info to {}", fileName, e); } } @@ -175,7 +175,7 @@ public static Optional getInfo(String name) { String content = new String(bytes, StandardCharsets.UTF_8); return Optional.of(content); } catch (IOException e) { - GlobalVariables.LOGGER.error("Error while reading info from {}", fileName, e); + Constants.LOGGER.error("Error while reading info from {}", fileName, e); return Optional.empty(); } } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/loader/LoaderManager.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/loader/LoaderManager.java index fb8a9337c..1d0e5365c 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/loader/LoaderManager.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/loader/LoaderManager.java @@ -1,9 +1,6 @@ package pl.skidam.automodpack_loader_core.loader; import pl.skidam.automodpack_core.loader.LoaderManagerService; -import pl.skidam.automodpack_core.utils.FileInspection; - -import java.util.Collection; public class LoaderManager implements LoaderManagerService { @@ -17,11 +14,6 @@ public boolean isModLoaded(String modId) { throw new AssertionError("Loader class not found"); } - @Override - public Collection getModList() { - throw new AssertionError("Loader class not found"); - } - @Override public String getLoaderVersion() { throw new AssertionError("Loader class not found"); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/mods/ModpackLoader.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/mods/ModpackLoader.java index e01e2b14e..8328b99a7 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/mods/ModpackLoader.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/mods/ModpackLoader.java @@ -2,6 +2,7 @@ import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.List; @@ -13,7 +14,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { throw new AssertionError("Loader class not found"); } } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/CurseForgeAPI.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/CurseForgeAPI.java index 484f45f43..d7e38f92e 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/CurseForgeAPI.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/CurseForgeAPI.java @@ -9,7 +9,7 @@ import java.util.List; import java.util.Map; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public record CurseForgeAPI(String requestUrl, String downloadUrl, String fileVersion, String fileName, String fileSize, String releaseType, String murmurHash, String sha1Hash) { diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/ModrinthAPI.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/ModrinthAPI.java index 183a9d0eb..302de8c62 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/ModrinthAPI.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/platforms/ModrinthAPI.java @@ -9,7 +9,7 @@ import java.util.LinkedList; import java.util.List; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public record ModrinthAPI(String modrinthID, String requestUrl, String downloadUrl, String fileVersion, String fileName, long fileSize, String releaseType, String SHA1Hash) { diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java index b1cb60e4e..9a10646c2 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java @@ -1,7 +1,7 @@ package pl.skidam.automodpack_loader_core.utils; import pl.skidam.automodpack_core.protocol.NetUtils; -import pl.skidam.automodpack_core.utils.CustomFileUtils; +import pl.skidam.automodpack_core.utils.SmartFileUtils; import pl.skidam.automodpack_core.utils.CustomThreadFactoryBuilder; import pl.skidam.automodpack_core.utils.FileInspection; import pl.skidam.automodpack_core.protocol.DownloadClient; @@ -15,7 +15,7 @@ import java.util.concurrent.*; import java.util.zip.GZIPInputStream; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class DownloadManager { private static final int MAX_DOWNLOADS_IN_PROGRESS = 5; @@ -38,7 +38,7 @@ public DownloadManager(long bytesToDownload) { // TODO: make caching system which detects if the same file was downloaded before and if so copy it instead of downloading again public void attachDownloadClient(DownloadClient downloadClient) { - this.downloadClient = downloadClient; + this.downloadClient = downloadClient; } public void download(Path file, String sha1, List urls, Runnable successCallback, Runnable failureCallback) { @@ -79,7 +79,7 @@ private void downloadTask(FileInspection.HashPathPair hashPathPair, QueuedDownlo boolean failed = true; if (Files.exists(queuedDownload.file)) { - String hash = CustomFileUtils.getHash(queuedDownload.file); + String hash = SmartFileUtils.getHash(queuedDownload.file); if (Objects.equals(hash, hashPathPair.hash())) { // Runs on success @@ -93,7 +93,7 @@ private void downloadTask(FileInspection.HashPathPair hashPathPair, QueuedDownlo if (failed) { bytesToDownload += queuedDownload.file.toFile().length(); // Add size of the whole file again because we will try to download it again - CustomFileUtils.executeOrder66(queuedDownload.file); + SmartFileUtils.executeOrder66(queuedDownload.file); if (!interrupted) { if (queuedDownload.attempts < (numberOfIndexes + 1) * MAX_DOWNLOAD_ATTEMPTS) { @@ -139,14 +139,14 @@ private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, QueuedDo Path outFile = queuedDownload.file; if (Files.exists(outFile)) { - if (Objects.equals(hashPathPair.hash(), CustomFileUtils.getHash(outFile))) { + if (Objects.equals(hashPathPair.hash(), SmartFileUtils.getHash(outFile))) { return; } else { - CustomFileUtils.executeOrder66(outFile); + SmartFileUtils.executeOrder66(outFile); } } - CustomFileUtils.setupFilePaths(outFile); + SmartFileUtils.setupFilePaths(outFile); var future = downloadClient.downloadFile(hashPathPair.hash().getBytes(StandardCharsets.UTF_8), outFile, (bytes) -> { bytesDownloaded += bytes; @@ -160,25 +160,26 @@ private void httpDownloadFile(String url, FileInspection.HashPathPair hashPathPa Path outFile = queuedDownload.file; if (Files.exists(outFile)) { - if (Objects.equals(hashPathPair.hash(), CustomFileUtils.getHash(outFile))) { + if (Objects.equals(hashPathPair.hash(), SmartFileUtils.getHash(outFile))) { return; } else { - CustomFileUtils.executeOrder66(outFile); + SmartFileUtils.executeOrder66(outFile); } } - CustomFileUtils.setupFilePaths(outFile); + SmartFileUtils.setupFilePaths(outFile); URLConnection connection = getHttpConnection(url); - try (OutputStream outputStream = new FileOutputStream(outFile.toFile()); - InputStream rawInputStream = new BufferedInputStream(connection.getInputStream(), NetUtils.DEFAULT_CHUNK_SIZE); - InputStream inputStream = ("gzip".equals(connection.getHeaderField("Content-Encoding"))) ? - new GZIPInputStream(rawInputStream) : rawInputStream) { + InputStream rawInputStream = connection.getInputStream(); + InputStream inputStream = ("gzip".equals(connection.getHeaderField("Content-Encoding"))) ? new GZIPInputStream(rawInputStream) : rawInputStream; + + try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outFile.toFile()), 64 * 1024); + InputStream is = inputStream) { byte[] buffer = new byte[NetUtils.DEFAULT_CHUNK_SIZE]; int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { + while ((bytesRead = is.read(buffer)) != -1) { bytesDownloaded += bytesRead; speedMeter.addDownloadedBytes(bytesRead); outputStream.write(buffer, 0, bytesRead); @@ -191,7 +192,6 @@ private void httpDownloadFile(String url, FileInspection.HashPathPair hashPathPa } private URLConnection getHttpConnection(String url) throws IOException { - LOGGER.info("Downloading from {}", url); URL connectionUrl = new URL(url); @@ -250,7 +250,7 @@ public void cancelAllAndShutdown() { queuedDownloads.clear(); downloadsInProgress.forEach((url, downloadData) -> { downloadData.future.cancel(true); - CustomFileUtils.executeOrder66(downloadData.file); + SmartFileUtils.executeOrder66(downloadData.file); }); // TODO Release the number of occupied permits, not all @@ -295,4 +295,4 @@ public String getFileName() { return file.getFileName().toString(); } } -} +} \ No newline at end of file diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/FetchManager.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/FetchManager.java index 3d19baf6d..04aaa27e3 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/FetchManager.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/FetchManager.java @@ -7,7 +7,7 @@ import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class FetchManager { diff --git a/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java b/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java index b8973977f..84cf28f7b 100644 --- a/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java +++ b/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java @@ -13,15 +13,15 @@ import net.fabricmc.loader.impl.util.SystemProperties; import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.loader.ModpackLoaderService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import pl.skidam.automodpack_loader_core_fabric.FabricLanguageAdapter; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import static pl.skidam.automodpack_loader_core_fabric.FabricLoaderImplAccessor.*; @SuppressWarnings({"unchecked", "unused"}) @@ -59,7 +59,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { Path modpackModsDir = modpackDir.resolve("mods"); Path standardModsDir = MODS_DIR; @@ -137,7 +137,7 @@ public List getModpackNestedConflicts(Path modpackDir) { List conflictingNestedMods = new ArrayList<>(); for (ModCandidate mod : conflictingNestedModsImpl) { - // Check mods provides, if theres some mod which is named with the same id as some other mod 'provides' remove the mod which provides that id as well, otherwise loader will crash + // Check mods provides, if there's some mod which is named with the same id as some other mod 'provides' remove the mod which provides that id as well, otherwise loader will crash if (originModIds.stream().anyMatch(mod.getProvides()::contains)) continue; @@ -148,7 +148,7 @@ public List getModpackNestedConflicts(Path modpackDir) { if (!Files.exists(path)) continue; - String hash = ClientCacheUtils.computeHashIfNeeded(path); + String hash = cache.getHashOrNull(path); if (hash == null) continue; diff --git a/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java b/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java index 14d8e85b1..c4ff76621 100644 --- a/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java +++ b/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java @@ -13,15 +13,15 @@ import net.fabricmc.loader.impl.util.SystemProperties; import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.loader.ModpackLoaderService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import pl.skidam.automodpack_loader_core_fabric.FabricLanguageAdapter; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import static pl.skidam.automodpack_loader_core_fabric.FabricLoaderImplAccessor.*; @SuppressWarnings({"unchecked", "unused"}) @@ -59,7 +59,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { Path modpackModsDir = modpackDir.resolve("mods"); Path standardModsDir = MODS_DIR; @@ -148,7 +148,7 @@ public List getModpackNestedConflicts(Path modpackDir) { if (!Files.exists(path)) continue; - String hash = ClientCacheUtils.computeHashIfNeeded(path); + String hash = cache.getHashOrNull(path); if (hash == null) continue; diff --git a/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/loader/LoaderManager.java b/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/loader/LoaderManager.java index d1a68c20e..2dc80910f 100644 --- a/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/loader/LoaderManager.java +++ b/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/loader/LoaderManager.java @@ -3,19 +3,11 @@ import net.fabricmc.api.EnvType; import net.fabricmc.loader.api.FabricLoader; import net.fabricmc.loader.api.ModContainer; -import net.fabricmc.loader.api.metadata.ModDependency; import net.fabricmc.loader.api.metadata.ModEnvironment; import pl.skidam.automodpack_core.loader.LoaderManagerService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; -import pl.skidam.automodpack_core.utils.FileInspection; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; - @SuppressWarnings("unused") public class LoaderManager implements LoaderManagerService { @@ -29,76 +21,12 @@ public boolean isModLoaded(String modId) { return FabricLoader.getInstance().isModLoaded(modId); } - private Collection modList = new ArrayList<>(); - private int lastLoadingModListSize = -1; - - @Override - public Collection getModList() { - - Collection mods = FabricLoader.getInstance().getAllMods(); - - if (!modList.isEmpty() && lastLoadingModListSize == mods.size()) { - return modList; - } - - lastLoadingModListSize = mods.size(); - Collection modList = new ArrayList<>(); - - for (var info : mods) { - try { - String modID = info.getMetadata().getId(); - Path path = getModPath(modID); - if (path == null || path.toString().isEmpty()) // If we cant get the path, we skip the mod, its probably JiJed, we dont need it in the list - continue; - - if (!Files.exists(path)) - continue; - - String hash = ClientCacheUtils.computeHashIfNeeded(path); - if (hash == null) - continue; - - Set providesIDs = new HashSet<>(info.getMetadata().getProvides()); - List dependencies = info.getMetadata().getDependencies().stream().filter(d -> d.getKind().equals(ModDependency.Kind.DEPENDS)).map(ModDependency::getModId).toList(); - - FileInspection.Mod mod = new FileInspection.Mod( - modID, - hash, - providesIDs, - info.getMetadata().getVersion().getFriendlyString(), - path, - getModEnvironment(modID), - dependencies); - - modList.add(mod); - } catch (Exception ignored) {} - } - - return this.modList = modList; - } - @Override public String getLoaderVersion() { Optional modContainer = FabricLoader.getInstance().getModContainer("fabricloader"); return modContainer.map(container -> container.getMetadata().getVersion().getFriendlyString()).orElse(null); } - private Path getModPath(String modId) { - if (!isModLoaded(modId)) return null; - - try { - for (ModContainer modContainer : FabricLoader.getInstance().getAllMods()) { - if (modContainer.getMetadata().getId().equals(modId)) { - FileSystem fileSys = modContainer.getRootPaths().get(0).getFileSystem(); - return Path.of(fileSys.toString()); - } - } - } catch (Exception ignored) {} - - LOGGER.error("Could not find jar file for {}", modId); - return null; - } - @Override public EnvironmentType getEnvironmentType() { if (FabricLoader.getInstance().getEnvironmentType() == EnvType.CLIENT) { diff --git a/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/mods/ModpackLoader.java b/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/mods/ModpackLoader.java index 0f630efc8..c8c25815c 100644 --- a/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/mods/ModpackLoader.java +++ b/loader/fabric/core/src/main/java/pl/skidam/automodpack_loader_core_fabric/mods/ModpackLoader.java @@ -3,13 +3,14 @@ import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; import pl.skidam.automodpack_core.utils.SemanticVersion; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import pl.skidam.automodpack_loader_core_fabric_15.mods.ModpackLoader15; import pl.skidam.automodpack_loader_core_fabric_16.mods.ModpackLoader16; import java.nio.file.Path; import java.util.List; -import static pl.skidam.automodpack_core.GlobalVariables.LOADER_MANAGER; +import static pl.skidam.automodpack_core.Constants.LOADER_MANAGER; @SuppressWarnings("unused") public class ModpackLoader implements ModpackLoaderService { @@ -26,7 +27,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { - return INSTANCE.getModpackNestedConflicts(modpackDir); + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { + return INSTANCE.getModpackNestedConflicts(modpackDir, cache); } } \ No newline at end of file diff --git a/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java b/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java index 190ac1065..529a0b600 100644 --- a/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java +++ b/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java @@ -2,20 +2,10 @@ import net.minecraftforge.api.distmarker.Dist; import net.minecraftforge.fml.loading.FMLLoader; -import net.minecraftforge.fml.loading.moddiscovery.ModFileInfo; import net.minecraftforge.fml.loading.moddiscovery.ModInfo; -import net.minecraftforge.forgespi.language.IModInfo; import pl.skidam.automodpack_core.loader.LoaderManagerService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; -import pl.skidam.automodpack_core.utils.FileInspection; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static pl.skidam.automodpack_core.GlobalVariables.preload; +import static pl.skidam.automodpack_core.Constants.preload; @SuppressWarnings("unused") public class LoaderManager implements LoaderManagerService { @@ -30,79 +20,11 @@ public boolean isModLoaded(String modId) { return FMLLoader.getLoadingModList().getModFileById(modId) != null; } - private Collection modList = new ArrayList<>(); - private int lastLoadingModListSize = -1; - - @Override - public Collection getModList() { - - if (preload) { - return modList; - } - - List modInfo = FMLLoader.getLoadingModList().getMods(); - - if (!modList.isEmpty() && lastLoadingModListSize == modInfo.size()) { - // return cached - return modList; - } - - lastLoadingModListSize = modInfo.size(); - Collection modList = new ArrayList<>(); - - for (ModInfo info: modInfo) { - try { - String modID = info.getModId(); - Path path = getModPath(modID); - if (path == null || path.toString().isEmpty()) // If we cant get the path, we skip the mod, its probably JiJed, we dont need it in the list - continue; - - if (!Files.exists(path)) - continue; - - String hash = ClientCacheUtils.computeHashIfNeeded(path); - if (hash == null) - continue; - - List dependencies = info.getDependencies().stream().filter(IModInfo.ModVersion::isMandatory).map(IModInfo.ModVersion::getModId).toList(); - FileInspection.Mod mod = new FileInspection.Mod( - modID, - hash, - List.of(), - info.getOwningFile().versionString(), - path, - EnvironmentType.UNIVERSAL, - dependencies); - - modList.add(mod); - } catch (Exception ignored) {} - } - - return this.modList = modList; - } - @Override public String getLoaderVersion() { return FMLLoader.versionInfo().forgeVersion(); } - private Path getModPath(String modId) { - if (isDevelopmentEnvironment()) { - return null; - } - - if (isModLoaded(modId)) { - ModFileInfo modInfo = FMLLoader.getLoadingModList().getModFileById(modId); - - List mods = modInfo.getMods(); - if (!mods.isEmpty()) { - return mods.get(0).getOwningFile().getFile().getFilePath().toAbsolutePath(); - } - } - - return null; - } - @Override public EnvironmentType getEnvironmentType() { if (FMLLoader.getDist() == Dist.CLIENT) { diff --git a/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java b/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java index 013504a9d..756ba64e9 100644 --- a/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java +++ b/loader/forge/fml40/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java @@ -2,13 +2,14 @@ import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class ModpackLoader implements ModpackLoaderService { public static String CONNECTOR_MODS_PROPERTY = "connector.additionalModLocations"; @@ -33,7 +34,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { return new ArrayList<>(); } } diff --git a/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java b/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java index 6a2e32610..0b53525b8 100644 --- a/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java +++ b/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/loader/LoaderManager.java @@ -2,20 +2,10 @@ import net.minecraftforge.api.distmarker.Dist; import net.minecraftforge.fml.loading.FMLLoader; -import net.minecraftforge.fml.loading.moddiscovery.ModFileInfo; import net.minecraftforge.fml.loading.moddiscovery.ModInfo; -import net.minecraftforge.forgespi.language.IModInfo; import pl.skidam.automodpack_core.loader.LoaderManagerService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; -import pl.skidam.automodpack_core.utils.FileInspection; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; @SuppressWarnings("unused") public class LoaderManager implements LoaderManagerService { @@ -30,79 +20,11 @@ public boolean isModLoaded(String modId) { return FMLLoader.getLoadingModList().getModFileById(modId) != null; } - private Collection modList = new ArrayList<>(); - private int lastLoadingModListSize = -1; - - @Override - public Collection getModList() { - - if (preload) { - return modList; - } - - List modInfo = FMLLoader.getLoadingModList().getMods(); - - if (!modList.isEmpty() && lastLoadingModListSize == modInfo.size()) { - // return cached - return modList; - } - - lastLoadingModListSize = modInfo.size(); - Collection modList = new ArrayList<>(); - - for (ModInfo info : modInfo) { - try { - String modID = info.getModId(); - Path path = getModPath(modID); - if (path == null || path.toString().isEmpty()) // If we cant get the path, we skip the mod, its probably JiJed, we dont need it in the list - continue; - - if (!Files.exists(path)) - continue; - - String hash = ClientCacheUtils.computeHashIfNeeded(path); - if (hash == null) - continue; - - List dependencies = info.getDependencies().stream().filter(IModInfo.ModVersion::isMandatory).map(IModInfo.ModVersion::getModId).toList(); - FileInspection.Mod mod = new FileInspection.Mod( - modID, - hash, - List.of(), - info.getOwningFile().versionString(), - path, - EnvironmentType.UNIVERSAL, - dependencies); - modList.add(mod); - } catch (Exception ignored) { - } - } - - return this.modList = modList; - } - @Override public String getLoaderVersion() { return FMLLoader.versionInfo().forgeVersion(); } - private Path getModPath(String modId) { - if (isDevelopmentEnvironment()) { - return null; - } - - if (isModLoaded(modId)) { - ModFileInfo modInfo = FMLLoader.getLoadingModList().getModFileById(modId); - - List mods = modInfo.getMods(); - if (!mods.isEmpty()) { - return mods.get(0).getOwningFile().getFile().getFilePath().toAbsolutePath(); - } - } - - return null; - } - @Override public EnvironmentType getEnvironmentType() { if (FMLLoader.getDist() == Dist.CLIENT) { diff --git a/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java b/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java index a1f876af4..2cb88bb0e 100644 --- a/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java +++ b/loader/forge/fml47/src/main/java/pl/skidam/automodpack_loader_core_forge/mods/ModpackLoader.java @@ -2,13 +2,14 @@ import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class ModpackLoader implements ModpackLoaderService { public static String CONNECTOR_MODS_PROPERTY = "connector.additionalModLocations"; @@ -33,7 +34,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { return new ArrayList<>(); } } diff --git a/loader/loader-fabric-core.gradle.kts b/loader/loader-fabric-core.gradle.kts index 082ff46e1..d628c684f 100644 --- a/loader/loader-fabric-core.gradle.kts +++ b/loader/loader-fabric-core.gradle.kts @@ -36,6 +36,7 @@ dependencies { implementation("io.netty:netty-codec-haproxy:4.2.9.Final") { isTransitive = false } + implementation("com.h2database:h2-mvstore:2.4.240") } configurations { diff --git a/loader/loader-forge.gradle.kts b/loader/loader-forge.gradle.kts index d356185c7..674ee0cee 100644 --- a/loader/loader-forge.gradle.kts +++ b/loader/loader-forge.gradle.kts @@ -36,6 +36,7 @@ dependencies { implementation("io.netty:netty-codec-haproxy:4.2.9.Final") { isTransitive = false } + implementation("com.h2database:h2-mvstore:2.4.240") } configurations { diff --git a/loader/loader-neoforge.gradle.kts b/loader/loader-neoforge.gradle.kts index de19c56c9..9689beccc 100644 --- a/loader/loader-neoforge.gradle.kts +++ b/loader/loader-neoforge.gradle.kts @@ -36,6 +36,7 @@ dependencies { implementation("io.netty:netty-codec-haproxy:4.2.9.Final") { isTransitive = false } + implementation("com.h2database:h2-mvstore:2.4.240") } configurations { diff --git a/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java b/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java index f21de22c7..77d4c2918 100644 --- a/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java +++ b/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java @@ -3,20 +3,10 @@ import net.neoforged.api.distmarker.Dist; import net.neoforged.fml.loading.FMLLoader; import net.neoforged.fml.loading.LoadingModList; -import net.neoforged.fml.loading.moddiscovery.ModFileInfo; import net.neoforged.fml.loading.moddiscovery.ModInfo; -import net.neoforged.neoforgespi.language.IModInfo; import pl.skidam.automodpack_core.loader.LoaderManagerService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; -import pl.skidam.automodpack_core.utils.FileInspection; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; @SuppressWarnings("unused") public class LoaderManager implements LoaderManagerService { @@ -37,80 +27,11 @@ public boolean isModLoaded(String modId) { return loadingModList.getModFileById(modId) != null; } - private Collection modList = new ArrayList<>(); - private int lastLoadingModListSize = -1; - - // Does it even still make sense to have this? FileInspection class should do everything anyway - @Override - public Collection getModList() { - - if (preload) { // always empty on preload - return modList; - } - - List modInfo = FMLLoader.getCurrent().getLoadingModList().getMods(); - - if (!modList.isEmpty() && lastLoadingModListSize == modInfo.size()) { - // return cached - return modList; - } - - lastLoadingModListSize = modInfo.size(); - Collection modList = new ArrayList<>(); - - for (ModInfo info: modInfo) { - try { - String modID = info.getModId(); - Path path = getModPath(modID); - if (path == null || path.toString().isEmpty()) // If we cant get the path, we skip the mod, its probably JiJed, we dont need it in the list - continue; - - if (!Files.exists(path)) - continue; - - String hash = ClientCacheUtils.computeHashIfNeeded(path); - if (hash == null) - continue; - - List dependencies = info.getDependencies().stream().filter(d -> d.getType() == IModInfo.DependencyType.REQUIRED).map(IModInfo.ModVersion::getModId).toList(); - FileInspection.Mod mod = new FileInspection.Mod( - modID, - hash, - List.of(), - info.getOwningFile().versionString(), - path, - EnvironmentType.UNIVERSAL, - dependencies); - - modList.add(mod); - } catch (Exception ignored) {} - } - - return this.modList = modList; - } - @Override public String getLoaderVersion() { return FMLLoader.getCurrent().getVersionInfo().neoForgeVersion(); } - private Path getModPath(String modId) { - if (isDevelopmentEnvironment()) { - return null; - } - - if (isModLoaded(modId)) { - ModFileInfo modInfo = FMLLoader.getCurrent().getLoadingModList().getModFileById(modId); - - List mods = modInfo.getMods(); - if (!mods.isEmpty()) { - return mods.getFirst().getOwningFile().getFile().getFilePath().toAbsolutePath(); - } - } - - return null; - } - @Override public EnvironmentType getEnvironmentType() { if (FMLLoader.getCurrent().getDist() == Dist.CLIENT) { diff --git a/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java b/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java index d95cf15c9..2dc4f71be 100644 --- a/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java +++ b/loader/neoforge/fml10/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java @@ -2,13 +2,14 @@ import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class ModpackLoader implements ModpackLoaderService { public static String CONNECTOR_MODS_PROPERTY = "connector.additionalModLocations"; @@ -33,7 +34,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { return new ArrayList<>(); } } diff --git a/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java b/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java index 03b3c0324..13083e566 100644 --- a/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java +++ b/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java @@ -3,20 +3,10 @@ import net.neoforged.api.distmarker.Dist; import net.neoforged.fml.loading.FMLLoader; import net.neoforged.fml.loading.LoadingModList; -import net.neoforged.fml.loading.moddiscovery.ModFileInfo; import net.neoforged.fml.loading.moddiscovery.ModInfo; -import net.neoforged.neoforgespi.language.IModInfo; import pl.skidam.automodpack_core.loader.LoaderManagerService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; -import pl.skidam.automodpack_core.utils.FileInspection; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; @SuppressWarnings("unused") public class LoaderManager implements LoaderManagerService { @@ -37,80 +27,11 @@ public boolean isModLoaded(String modId) { return loadingModList.getModFileById(modId) != null; } - private Collection modList = new ArrayList<>(); - private int lastLoadingModListSize = -1; - - // Does it even still make sense to have this? FileInspection class should do everything anyway - @Override - public Collection getModList() { - - if (preload) { // always empty on preload - return modList; - } - - List modInfo = FMLLoader.getLoadingModList().getMods(); - - if (!modList.isEmpty() && lastLoadingModListSize == modInfo.size()) { - // return cached - return modList; - } - - lastLoadingModListSize = modInfo.size(); - Collection modList = new ArrayList<>(); - - for (ModInfo info: modInfo) { - try { - String modID = info.getModId(); - Path path = getModPath(modID); - if (path == null || path.toString().isEmpty()) // If we cant get the path, we skip the mod, its probably JiJed, we dont need it in the list - continue; - - if (!Files.exists(path)) - continue; - - String hash = ClientCacheUtils.computeHashIfNeeded(path); - if (hash == null) - continue; - - List dependencies = info.getDependencies().stream().filter(d -> d.getType() == IModInfo.DependencyType.REQUIRED).map(IModInfo.ModVersion::getModId).toList(); - FileInspection.Mod mod = new FileInspection.Mod( - modID, - hash, - List.of(), - info.getOwningFile().versionString(), - path, - EnvironmentType.UNIVERSAL, - dependencies); - - modList.add(mod); - } catch (Exception ignored) {} - } - - return this.modList = modList; - } - @Override public String getLoaderVersion() { return FMLLoader.versionInfo().neoForgeVersion(); } - private Path getModPath(String modId) { - if (isDevelopmentEnvironment()) { - return null; - } - - if (isModLoaded(modId)) { - ModFileInfo modInfo = FMLLoader.getLoadingModList().getModFileById(modId); - - List mods = modInfo.getMods(); - if (!mods.isEmpty()) { - return mods.getFirst().getOwningFile().getFile().getFilePath().toAbsolutePath(); - } - } - - return null; - } - @Override public EnvironmentType getEnvironmentType() { if (FMLLoader.getDist() == Dist.CLIENT) { diff --git a/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java b/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java index d95cf15c9..2dc4f71be 100644 --- a/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java +++ b/loader/neoforge/fml2/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java @@ -2,13 +2,14 @@ import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class ModpackLoader implements ModpackLoaderService { public static String CONNECTOR_MODS_PROPERTY = "connector.additionalModLocations"; @@ -33,7 +34,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { return new ArrayList<>(); } } diff --git a/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java b/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java index 03b3c0324..13083e566 100644 --- a/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java +++ b/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/loader/LoaderManager.java @@ -3,20 +3,10 @@ import net.neoforged.api.distmarker.Dist; import net.neoforged.fml.loading.FMLLoader; import net.neoforged.fml.loading.LoadingModList; -import net.neoforged.fml.loading.moddiscovery.ModFileInfo; import net.neoforged.fml.loading.moddiscovery.ModInfo; -import net.neoforged.neoforgespi.language.IModInfo; import pl.skidam.automodpack_core.loader.LoaderManagerService; -import pl.skidam.automodpack_core.utils.ClientCacheUtils; -import pl.skidam.automodpack_core.utils.FileInspection; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; @SuppressWarnings("unused") public class LoaderManager implements LoaderManagerService { @@ -37,80 +27,11 @@ public boolean isModLoaded(String modId) { return loadingModList.getModFileById(modId) != null; } - private Collection modList = new ArrayList<>(); - private int lastLoadingModListSize = -1; - - // Does it even still make sense to have this? FileInspection class should do everything anyway - @Override - public Collection getModList() { - - if (preload) { // always empty on preload - return modList; - } - - List modInfo = FMLLoader.getLoadingModList().getMods(); - - if (!modList.isEmpty() && lastLoadingModListSize == modInfo.size()) { - // return cached - return modList; - } - - lastLoadingModListSize = modInfo.size(); - Collection modList = new ArrayList<>(); - - for (ModInfo info: modInfo) { - try { - String modID = info.getModId(); - Path path = getModPath(modID); - if (path == null || path.toString().isEmpty()) // If we cant get the path, we skip the mod, its probably JiJed, we dont need it in the list - continue; - - if (!Files.exists(path)) - continue; - - String hash = ClientCacheUtils.computeHashIfNeeded(path); - if (hash == null) - continue; - - List dependencies = info.getDependencies().stream().filter(d -> d.getType() == IModInfo.DependencyType.REQUIRED).map(IModInfo.ModVersion::getModId).toList(); - FileInspection.Mod mod = new FileInspection.Mod( - modID, - hash, - List.of(), - info.getOwningFile().versionString(), - path, - EnvironmentType.UNIVERSAL, - dependencies); - - modList.add(mod); - } catch (Exception ignored) {} - } - - return this.modList = modList; - } - @Override public String getLoaderVersion() { return FMLLoader.versionInfo().neoForgeVersion(); } - private Path getModPath(String modId) { - if (isDevelopmentEnvironment()) { - return null; - } - - if (isModLoaded(modId)) { - ModFileInfo modInfo = FMLLoader.getLoadingModList().getModFileById(modId); - - List mods = modInfo.getMods(); - if (!mods.isEmpty()) { - return mods.getFirst().getOwningFile().getFile().getFilePath().toAbsolutePath(); - } - } - - return null; - } - @Override public EnvironmentType getEnvironmentType() { if (FMLLoader.getDist() == Dist.CLIENT) { diff --git a/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java b/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java index d95cf15c9..2dc4f71be 100644 --- a/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java +++ b/loader/neoforge/fml4/src/main/java/pl/skidam/automodpack_loader_core_neoforge/mods/ModpackLoader.java @@ -2,13 +2,14 @@ import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; public class ModpackLoader implements ModpackLoaderService { public static String CONNECTOR_MODS_PROPERTY = "connector.additionalModLocations"; @@ -33,7 +34,7 @@ public void loadModpack(List modpackMods) { } @Override - public List getModpackNestedConflicts(Path modpackDir) { + public List getModpackNestedConflicts(Path modpackDir, FileMetadataCache cache) { return new ArrayList<>(); } } diff --git a/src/main/java/pl/skidam/automodpack/client/audio/AudioManager.java b/src/main/java/pl/skidam/automodpack/client/audio/AudioManager.java index 5b7418f72..845f28ec5 100644 --- a/src/main/java/pl/skidam/automodpack/client/audio/AudioManager.java +++ b/src/main/java/pl/skidam/automodpack/client/audio/AudioManager.java @@ -14,12 +14,12 @@ /*? if neoforge {*/ /*import net.neoforged.bus.api.IEventBus; import net.neoforged.neoforge.registries.DeferredRegister; -import static pl.skidam.automodpack_core.GlobalVariables.MOD_ID; +import static pl.skidam.automodpack_core.Constants.MOD_ID; *//*?} else if forge {*/ /*import net.minecraftforge.eventbus.api.IEventBus; import net.minecraftforge.registries.DeferredRegister; import net.minecraftforge.registries.ForgeRegistries; -import static pl.skidam.automodpack_core.GlobalVariables.MOD_ID; +import static pl.skidam.automodpack_core.Constants.MOD_ID; *//*?} else {*/ /*? if >=1.19.3 {*/ import net.minecraft.core.Registry; diff --git a/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java b/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java index b81a3e7b8..aac46d342 100644 --- a/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java +++ b/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java @@ -1,7 +1,7 @@ package pl.skidam.automodpack.client.ui; -import static pl.skidam.automodpack_core.GlobalVariables.clientConfig; -import static pl.skidam.automodpack_core.GlobalVariables.clientConfigFile; +import static pl.skidam.automodpack_core.Constants.clientConfig; +import static pl.skidam.automodpack_core.Constants.clientConfigFile; import net.minecraft.ChatFormatting; import net.minecraft.util.Util; diff --git a/src/main/java/pl/skidam/automodpack/client/ui/FingerprintVerificationScreen.java b/src/main/java/pl/skidam/automodpack/client/ui/FingerprintVerificationScreen.java index 507970a06..6391c039f 100644 --- a/src/main/java/pl/skidam/automodpack/client/ui/FingerprintVerificationScreen.java +++ b/src/main/java/pl/skidam/automodpack/client/ui/FingerprintVerificationScreen.java @@ -10,7 +10,7 @@ import pl.skidam.automodpack.client.ui.versioned.VersionedMatrices; import pl.skidam.automodpack.client.ui.versioned.VersionedScreen; import pl.skidam.automodpack.client.ui.versioned.VersionedText; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; public class FingerprintVerificationScreen extends VersionedScreen { private final Screen parent; @@ -101,7 +101,7 @@ private void verifyFingerprint() { } validatedCallback.run(); } else { - GlobalVariables.LOGGER.error("Server fingerprint validation failed, try again"); + Constants.LOGGER.error("Server fingerprint validation failed, try again"); if (this.minecraft != null) { /*? if > 1.21.1 {*/ this.minecraft.getToastManager().addToast(failedToast); diff --git a/src/main/java/pl/skidam/automodpack/client/ui/SkipVerificationScreen.java b/src/main/java/pl/skidam/automodpack/client/ui/SkipVerificationScreen.java index dfe6e0f38..a06c6511a 100644 --- a/src/main/java/pl/skidam/automodpack/client/ui/SkipVerificationScreen.java +++ b/src/main/java/pl/skidam/automodpack/client/ui/SkipVerificationScreen.java @@ -10,7 +10,7 @@ import pl.skidam.automodpack.client.ui.versioned.VersionedMatrices; import pl.skidam.automodpack.client.ui.versioned.VersionedScreen; import pl.skidam.automodpack.client.ui.versioned.VersionedText; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; public class SkipVerificationScreen extends VersionedScreen { private final Screen verificationScreen; @@ -96,7 +96,7 @@ private void confirmSkip() { } validatedCallback.run(); } else { - GlobalVariables.LOGGER.error("Skip verification text mismatch, try again"); + Constants.LOGGER.error("Skip verification text mismatch, try again"); if (this.minecraft != null) { /*? if > 1.21.1 {*/ this.minecraft.getToastManager().addToast(failedToast); diff --git a/src/main/java/pl/skidam/automodpack/init/Common.java b/src/main/java/pl/skidam/automodpack/init/Common.java index b42a169fb..64caa5282 100644 --- a/src/main/java/pl/skidam/automodpack/init/Common.java +++ b/src/main/java/pl/skidam/automodpack/init/Common.java @@ -11,7 +11,7 @@ import java.util.HashMap; import java.util.Map; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class Common { diff --git a/src/main/java/pl/skidam/automodpack/init/FabricInit.java b/src/main/java/pl/skidam/automodpack/init/FabricInit.java index 662b55d80..92c675613 100644 --- a/src/main/java/pl/skidam/automodpack/init/FabricInit.java +++ b/src/main/java/pl/skidam/automodpack/init/FabricInit.java @@ -8,7 +8,7 @@ import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_loader_core.screen.ScreenManager; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import net.fabricmc.fabric.api.command./*? if <1.19.1 {*/ /*v1 *//*?} else {*/ v2 /*?}*/.CommandRegistrationCallback; public class FabricInit { diff --git a/src/main/java/pl/skidam/automodpack/init/ForgeInit.java b/src/main/java/pl/skidam/automodpack/init/ForgeInit.java index da20c9ac5..509adccd6 100644 --- a/src/main/java/pl/skidam/automodpack/init/ForgeInit.java +++ b/src/main/java/pl/skidam/automodpack/init/ForgeInit.java @@ -13,7 +13,7 @@ import net.minecraftforge.fml.javafmlmod.FMLJavaModLoadingContext; import net.minecraftforge.fml.common.Mod; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; @Mod(MOD_ID + "_mod") public class ForgeInit { diff --git a/src/main/java/pl/skidam/automodpack/init/NeoForgeInit.java b/src/main/java/pl/skidam/automodpack/init/NeoForgeInit.java index bfb5485bc..5a27ccbcd 100644 --- a/src/main/java/pl/skidam/automodpack/init/NeoForgeInit.java +++ b/src/main/java/pl/skidam/automodpack/init/NeoForgeInit.java @@ -16,7 +16,7 @@ import net.neoforged.fml.common.Mod; import net.neoforged.neoforge.event.RegisterCommandsEvent; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; @Mod(MOD_ID + "_mod") public class NeoForgeInit { diff --git a/src/main/java/pl/skidam/automodpack/loader/GameCall.java b/src/main/java/pl/skidam/automodpack/loader/GameCall.java index 538e89a47..803d62907 100644 --- a/src/main/java/pl/skidam/automodpack/loader/GameCall.java +++ b/src/main/java/pl/skidam/automodpack/loader/GameCall.java @@ -6,7 +6,7 @@ import java.net.SocketAddress; import static pl.skidam.automodpack.init.Common.server; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class GameCall implements GameCallService { diff --git a/src/main/java/pl/skidam/automodpack/mixin/core/LoginQueryRequestS2CPacketMixin.java b/src/main/java/pl/skidam/automodpack/mixin/core/LoginQueryRequestS2CPacketMixin.java index f2efe8ea5..15a325b36 100644 --- a/src/main/java/pl/skidam/automodpack/mixin/core/LoginQueryRequestS2CPacketMixin.java +++ b/src/main/java/pl/skidam/automodpack/mixin/core/LoginQueryRequestS2CPacketMixin.java @@ -13,7 +13,7 @@ import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; import pl.skidam.automodpack.networking.PayloadHelper; import pl.skidam.automodpack.networking.server.LoginRequestPayload; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; // TODO find better way to do this, its mixin only for 1.20.2 and above @Mixin(value = ClientboundCustomQueryPacket.class, priority = 300) @@ -28,7 +28,7 @@ public class LoginQueryRequestS2CPacketMixin { @Inject(method = "readPayload", at = @At("HEAD"), cancellable = true) private static void readPayload(Identifier id, FriendlyByteBuf buf, CallbackInfoReturnable cir) { - if (id.getNamespace().equals(GlobalVariables.MOD_ID)) { + if (id.getNamespace().equals(Constants.MOD_ID)) { cir.setReturnValue(new LoginRequestPayload(id, PayloadHelper.read(buf, MAX_PAYLOAD_SIZE))); } } diff --git a/src/main/java/pl/skidam/automodpack/mixin/core/PlayerManagerMixin.java b/src/main/java/pl/skidam/automodpack/mixin/core/PlayerManagerMixin.java index d4a870a3e..f07c762a9 100644 --- a/src/main/java/pl/skidam/automodpack/mixin/core/PlayerManagerMixin.java +++ b/src/main/java/pl/skidam/automodpack/mixin/core/PlayerManagerMixin.java @@ -24,7 +24,7 @@ import java.net.URI; /*?}*/ -import static pl.skidam.automodpack_core.GlobalVariables.serverConfig; +import static pl.skidam.automodpack_core.Constants.serverConfig; @Mixin(PlayerList.class) public class PlayerManagerMixin { diff --git a/src/main/java/pl/skidam/automodpack/mixin/core/ServerLoginNetworkHandlerMixin.java b/src/main/java/pl/skidam/automodpack/mixin/core/ServerLoginNetworkHandlerMixin.java index 14b18e657..73c4fbad9 100644 --- a/src/main/java/pl/skidam/automodpack/mixin/core/ServerLoginNetworkHandlerMixin.java +++ b/src/main/java/pl/skidam/automodpack/mixin/core/ServerLoginNetworkHandlerMixin.java @@ -9,7 +9,7 @@ import pl.skidam.automodpack.networking.client.LoginResponsePayload; import pl.skidam.automodpack.networking.server.ServerLoginNetworkAddon; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; @Mixin(value = ServerLoginPacketListenerImpl.class, priority = 300) public abstract class ServerLoginNetworkHandlerMixin { diff --git a/src/main/java/pl/skidam/automodpack/mixin/core/ServerNetworkIoMixin.java b/src/main/java/pl/skidam/automodpack/mixin/core/ServerNetworkIoMixin.java index 6de05e01c..f86ac04ca 100644 --- a/src/main/java/pl/skidam/automodpack/mixin/core/ServerNetworkIoMixin.java +++ b/src/main/java/pl/skidam/automodpack/mixin/core/ServerNetworkIoMixin.java @@ -5,10 +5,10 @@ import org.spongepowered.asm.mixin.injection.At; import org.spongepowered.asm.mixin.injection.Inject; import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; -import pl.skidam.automodpack_core.GlobalVariables; +import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.protocol.netty.handler.ProtocolServerHandler; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; @Mixin(targets = "net/minecraft/server/network/ServerConnectionListener$1", priority = 2137) public abstract class ServerNetworkIoMixin { @@ -30,6 +30,6 @@ private void injectAutoModpackHost(Channel channel, CallbackInfo ci) { return; } - channel.pipeline().addFirst(MOD_ID, new ProtocolServerHandler(GlobalVariables.hostServer.getSslCtx())); + channel.pipeline().addFirst(MOD_ID, new ProtocolServerHandler(Constants.hostServer.getSslCtx())); } } diff --git a/src/main/java/pl/skidam/automodpack/mixin/dev/TestButton.java b/src/main/java/pl/skidam/automodpack/mixin/dev/TestButton.java index ed2b888ae..5196eaf62 100644 --- a/src/main/java/pl/skidam/automodpack/mixin/dev/TestButton.java +++ b/src/main/java/pl/skidam/automodpack/mixin/dev/TestButton.java @@ -9,7 +9,7 @@ import pl.skidam.automodpack.client.ui.versioned.VersionedText; import pl.skidam.automodpack_loader_core.screen.ScreenManager; -import static pl.skidam.automodpack_core.GlobalVariables.LOADER_MANAGER; +import static pl.skidam.automodpack_core.Constants.LOADER_MANAGER; import net.minecraft.client.gui.screens.Screen; import net.minecraft.client.gui.screens.TitleScreen; diff --git a/src/main/java/pl/skidam/automodpack/modpack/Commands.java b/src/main/java/pl/skidam/automodpack/modpack/Commands.java index d8b939338..4338d9c3a 100644 --- a/src/main/java/pl/skidam/automodpack/modpack/Commands.java +++ b/src/main/java/pl/skidam/automodpack/modpack/Commands.java @@ -22,7 +22,7 @@ import pl.skidam.automodpack_core.config.ConfigUtils; import static net.minecraft.commands.Commands.literal; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class Commands { diff --git a/src/main/java/pl/skidam/automodpack/networking/LoginNetworkingIDs.java b/src/main/java/pl/skidam/automodpack/networking/LoginNetworkingIDs.java index 855426e54..dc94bf331 100644 --- a/src/main/java/pl/skidam/automodpack/networking/LoginNetworkingIDs.java +++ b/src/main/java/pl/skidam/automodpack/networking/LoginNetworkingIDs.java @@ -2,7 +2,7 @@ import pl.skidam.automodpack.init.Common; -import static pl.skidam.automodpack_core.GlobalVariables.MOD_ID; +import static pl.skidam.automodpack_core.Constants.MOD_ID; import net.minecraft.resources.Identifier; diff --git a/src/main/java/pl/skidam/automodpack/networking/LoginQueryParser.java b/src/main/java/pl/skidam/automodpack/networking/LoginQueryParser.java index 20930def0..b54584f82 100644 --- a/src/main/java/pl/skidam/automodpack/networking/LoginQueryParser.java +++ b/src/main/java/pl/skidam/automodpack/networking/LoginQueryParser.java @@ -7,7 +7,7 @@ import net.minecraft.network.protocol.login.custom.CustomQueryPayload; /*?}*/ -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; import net.minecraft.network.FriendlyByteBuf; import net.minecraft.network.protocol.Packet; diff --git a/src/main/java/pl/skidam/automodpack/networking/ModPackets.java b/src/main/java/pl/skidam/automodpack/networking/ModPackets.java index 42d475c01..11d6589c7 100644 --- a/src/main/java/pl/skidam/automodpack/networking/ModPackets.java +++ b/src/main/java/pl/skidam/automodpack/networking/ModPackets.java @@ -16,7 +16,7 @@ import java.net.InetSocketAddress; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class ModPackets { public static final Identifier HANDSHAKE = LoginNetworkingIDs.getResourceLocation(LoginNetworkingIDs.HANDSHAKE); diff --git a/src/main/java/pl/skidam/automodpack/networking/client/ClientLoginNetworkAddon.java b/src/main/java/pl/skidam/automodpack/networking/client/ClientLoginNetworkAddon.java index 1b21725d1..e12cdad5e 100644 --- a/src/main/java/pl/skidam/automodpack/networking/client/ClientLoginNetworkAddon.java +++ b/src/main/java/pl/skidam/automodpack/networking/client/ClientLoginNetworkAddon.java @@ -12,7 +12,7 @@ import net.minecraft.network.protocol.login.ServerboundCustomQueryAnswerPacket; import net.minecraft.resources.Identifier; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; // credits to fabric api public class ClientLoginNetworkAddon { diff --git a/src/main/java/pl/skidam/automodpack/networking/packet/DataC2SPacket.java b/src/main/java/pl/skidam/automodpack/networking/packet/DataC2SPacket.java index 3dd2f38ed..4360fee0f 100644 --- a/src/main/java/pl/skidam/automodpack/networking/packet/DataC2SPacket.java +++ b/src/main/java/pl/skidam/automodpack/networking/packet/DataC2SPacket.java @@ -23,7 +23,7 @@ import net.minecraft.client.multiplayer.ClientHandshakePacketListenerImpl; import net.minecraft.network.FriendlyByteBuf; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; import static pl.skidam.automodpack_core.config.ConfigTools.GSON; public class DataC2SPacket { diff --git a/src/main/java/pl/skidam/automodpack/networking/packet/DataS2CPacket.java b/src/main/java/pl/skidam/automodpack/networking/packet/DataS2CPacket.java index a4f02bab6..2f7acf163 100644 --- a/src/main/java/pl/skidam/automodpack/networking/packet/DataS2CPacket.java +++ b/src/main/java/pl/skidam/automodpack/networking/packet/DataS2CPacket.java @@ -13,7 +13,7 @@ import pl.skidam.automodpack.client.ui.versioned.VersionedText; import pl.skidam.automodpack.mixin.core.ServerLoginNetworkHandlerAccessor; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class DataS2CPacket { diff --git a/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeC2SPacket.java b/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeC2SPacket.java index f8d61035e..7e6bf5653 100644 --- a/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeC2SPacket.java +++ b/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeC2SPacket.java @@ -14,7 +14,7 @@ import net.minecraft.client.multiplayer.ClientHandshakePacketListenerImpl; import net.minecraft.network.FriendlyByteBuf; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class HandshakeC2SPacket { diff --git a/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeS2CPacket.java b/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeS2CPacket.java index 7d8100098..7f529768a 100644 --- a/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeS2CPacket.java +++ b/src/main/java/pl/skidam/automodpack/networking/packet/HandshakeS2CPacket.java @@ -23,7 +23,7 @@ import java.util.UUID; import static pl.skidam.automodpack.networking.ModPackets.DATA; -import static pl.skidam.automodpack_core.GlobalVariables.*; +import static pl.skidam.automodpack_core.Constants.*; public class HandshakeS2CPacket { diff --git a/src/main/java/pl/skidam/automodpack/networking/server/ServerLoginNetworkAddon.java b/src/main/java/pl/skidam/automodpack/networking/server/ServerLoginNetworkAddon.java index 210898449..6425c1869 100644 --- a/src/main/java/pl/skidam/automodpack/networking/server/ServerLoginNetworkAddon.java +++ b/src/main/java/pl/skidam/automodpack/networking/server/ServerLoginNetworkAddon.java @@ -20,7 +20,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Future; -import static pl.skidam.automodpack_core.GlobalVariables.LOGGER; +import static pl.skidam.automodpack_core.Constants.LOGGER; // credits to fabric api public class ServerLoginNetworkAddon implements PacketSender { From 6d3a981e22db6dbed62729b627cbbd46e5dd7d31 Mon Sep 17 00:00:00 2001 From: skidam Date: Sat, 24 Jan 2026 20:43:10 +0100 Subject: [PATCH 02/22] Integrate FileMetadataCache for ModpackContent generation and improve multithreading --- .../pl/skidam/automodpack_core/Server.java | 2 +- .../modpack/ModpackContent.java | 100 ++++++++---------- .../modpack/ModpackExecutor.java | 14 ++- .../protocol/netty/NettyServer.java | 6 +- .../automodpack_core/modpack/ModpackTest.java | 2 +- 5 files changed, 59 insertions(+), 65 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/Server.java b/core/src/main/java/pl/skidam/automodpack_core/Server.java index a1993b433..a843f1731 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/Server.java +++ b/core/src/main/java/pl/skidam/automodpack_core/Server.java @@ -60,7 +60,7 @@ public static void main(String[] args) { mainModpackDir.toFile().mkdirs(); ModpackExecutor modpackExecutor = new ModpackExecutor(); - ModpackContent modpackContent = new ModpackContent(serverConfig.modpackName, null, mainModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, modpackExecutor.getExecutor()); + ModpackContent modpackContent = new ModpackContent(serverConfig.modpackName, null, mainModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, modpackExecutor.getExecutor(), null); boolean generated = modpackExecutor.generateNew(modpackContent); if (generated) { diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java index ae37c1c87..7a144fef2 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java @@ -4,6 +4,7 @@ import pl.skidam.automodpack_core.config.Jsons; import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.utils.*; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.nio.file.Files; import java.nio.file.Path; @@ -12,9 +13,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; +import java.util.stream.Stream; import static pl.skidam.automodpack_core.Constants.*; -import static pl.skidam.automodpack_core.Constants.LOGGER; public class ModpackContent { public final Set list = ConcurrentHashMap.newKeySet(); @@ -25,22 +26,24 @@ public class ModpackContent { private final FileTreeScanner FORCE_COPY_FILES_TO_STANDARD_LOCATION; private final Path MODPACK_DIR; private final ThreadPoolExecutor CREATION_EXECUTOR; + private final FileMetadataCache metadataCache; private final Map sha1MurmurMapPreviousContent = new HashMap<>(); - public ModpackContent(String modpackName, Path cwd, Path modpackDir, Set syncedFiles, Set allowEditsInFiles, Set forceCopyFilesToStandardLocation, ThreadPoolExecutor CREATION_EXECUTOR) { + public ModpackContent(String modpackName, Path cwd, Path modpackDir, Set syncedFiles, Set allowEditsInFiles, Set forceCopyFilesToStandardLocation, ThreadPoolExecutor CREATION_EXECUTOR, FileMetadataCache metadataCache) { this.MODPACK_NAME = modpackName; this.MODPACK_DIR = modpackDir; + this.CREATION_EXECUTOR = CREATION_EXECUTOR; + this.metadataCache = metadataCache; Set directoriesToSearch = new HashSet<>(2); if (MODPACK_DIR != null) directoriesToSearch.add(MODPACK_DIR); if (cwd != null) { directoriesToSearch.add(cwd); - this.SYNCED_FILES_CARDS = new FileTreeScanner(syncedFiles, Set.of(cwd)); // Synced files should search only in cwd + this.SYNCED_FILES_CARDS = new FileTreeScanner(syncedFiles, Set.of(cwd)); } else { this.SYNCED_FILES_CARDS = new FileTreeScanner(syncedFiles, Set.of()); } this.EDITABLE_CARDS = new FileTreeScanner(allowEditsInFiles, directoriesToSearch); this.FORCE_COPY_FILES_TO_STANDARD_LOCATION = new FileTreeScanner(forceCopyFilesToStandardLocation, directoriesToSearch); - this.CREATION_EXECUTOR = CREATION_EXECUTOR; } public String getModpackName() { @@ -78,26 +81,36 @@ public boolean create() { previousContent.list.forEach(item -> sha1MurmurMapPreviousContent.put(item.sha1, item.murmur)); }); - List> creationFutures = new ArrayList<>(); + Map filesToProcess = new HashMap<>(); - // host-modpack generation - if (MODPACK_DIR != null) { - LOGGER.info("Syncing {}...", MODPACK_DIR.getFileName()); - try (var pathStream = Files.walk(MODPACK_DIR)) { - creationFutures.addAll(generateAsync(pathStream.toList())); + SYNCED_FILES_CARDS.getMatchedPaths().values().forEach(path -> filesToProcess.put(SmartFileUtils.formatPath(path, MODPACK_DIR), path)); - // Wait till finish - creationFutures.forEach((CompletableFuture::join)); - creationFutures.clear(); + if (MODPACK_DIR != null) { + try (Stream stream = Files.walk(MODPACK_DIR)) { // in case there any files with the same relative path, we prefer from MODPACK_DIR, this will override previous entries + stream.forEach(path -> filesToProcess.put(SmartFileUtils.formatPath(path, MODPACK_DIR), path)); } } - // synced files generation - creationFutures.addAll(generateAsync(SYNCED_FILES_CARDS.getMatchedPaths().values().stream().toList())); + List> futures = filesToProcess.entrySet().stream() + .map(entry -> CompletableFuture.supplyAsync(() -> { + try { + return generateContent(entry.getValue(), entry.getKey()); + } catch (Exception e) { + LOGGER.error("Error generating content for {}", entry.getValue(), e); + return null; + } + }, CREATION_EXECUTOR)) + .toList(); - // Wait till finish - creationFutures.forEach((CompletableFuture::join)); - creationFutures.clear(); + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + + for (var future : futures) { + Jsons.ModpackContentFields.ModpackContentItem item = future.join(); + if (item != null) { + list.add(item); + pathsMap.put(item.sha1, SmartFileUtils.getPathFromCWD(item.file)); + } + } if (list.isEmpty()) { LOGGER.warn("Modpack is empty!"); @@ -110,7 +123,7 @@ public boolean create() { saveModpackContent(computedFilesToDelete); if (hostServer != null) { - hostServer.addPaths(pathsMap); + hostServer.setPaths(pathsMap); } return true; @@ -121,7 +134,6 @@ public Optional getPreviousContent() { return optionalModpackContentFile.map(ConfigTools::loadModpackContent); } - public boolean loadPreviousContent() { var optionalPreviousModpackContent = getPreviousContent(); if (optionalPreviousModpackContent.isEmpty()) return false; @@ -143,16 +155,14 @@ public boolean loadPreviousContent() { } if (hostServer != null) { - hostServer.addPaths(pathsMap); + hostServer.setPaths(pathsMap); } - // set all new variables saveModpackContent(previousModpackContent.nonModpackFilesToDelete); return true; } - // This is important to make it synchronized otherwise it could corrupt the file and crash public synchronized void saveModpackContent(Set nonModpackFilesToDelete) { if (nonModpackFilesToDelete == null) { throw new IllegalArgumentException("filesToDelete is null"); @@ -172,20 +182,14 @@ public synchronized void saveModpackContent(Set> generateAsync(List files) { - List> futures = new ArrayList<>(); - for (int i = 0; i < files.size(); i += 6) { - List subList = files.subList(i, Math.min(files.size(), i + 6)); - futures.add(CompletableFuture.runAsync(() -> subList.forEach(this::generate), CREATION_EXECUTOR)); - } - - return futures; + public CompletableFuture replaceAsync(Path file) { + return CompletableFuture.runAsync(() -> replace(file), CREATION_EXECUTOR); } - private void generate(Path file) { + public void replace(Path file) { + remove(file); try { - Jsons.ModpackContentFields.ModpackContentItem item = generateContent(file); + Jsons.ModpackContentFields.ModpackContentItem item = generateContent(file, SmartFileUtils.formatPath(file, MODPACK_DIR)); if (item != null) { LOGGER.info("generated content for {}", item.file); synchronized (list) { @@ -194,21 +198,11 @@ private void generate(Path file) { pathsMap.put(item.sha1, file); } } catch (Exception e) { - LOGGER.error("Error while generating content for: " + file + " generated from: " + MODPACK_DIR, e); + LOGGER.error("Error while replacing content for: " + file, e); } } - public CompletableFuture replaceAsync(Path file) { - return CompletableFuture.runAsync(() -> replace(file), CREATION_EXECUTOR); - } - - public void replace(Path file) { - remove(file); - generate(file); - } - public void remove(Path file) { - String modpackFile = SmartFileUtils.formatPath(file, MODPACK_DIR); synchronized (list) { @@ -223,19 +217,17 @@ public void remove(Path file) { } } - // check if file is inside automodpack Dir or its sub-dirs, unless it's inside hostModpackDir with exception of hostModpackContentFile public static boolean isInnerFile(Path file) { Path normalizedFilePath = file.toAbsolutePath().normalize(); - boolean isInner = normalizedFilePath.startsWith(automodpackDir.toAbsolutePath().normalize()) && - !normalizedFilePath.startsWith(hostModpackDir.toAbsolutePath().normalize()); - if (!isInner && normalizedFilePath.equals(hostModpackContentFile.toAbsolutePath().normalize())) { + boolean isInner = normalizedFilePath.startsWith(automodpackDir.toAbsolutePath().normalize()) && !normalizedFilePath.startsWith(hostModpackDir.toAbsolutePath().normalize()); + if (!isInner && normalizedFilePath.equals(hostModpackContentFile.toAbsolutePath().normalize())) { // special case, since its inside hostModpackDir return true; } return isInner; } - private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path file) throws Exception { + private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path file, final String formattedFile) throws Exception { if (!Files.isRegularFile(file)) return null; if (serverConfig == null) { @@ -247,9 +239,6 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path return null; } - String formattedFile = SmartFileUtils.formatPath(file, MODPACK_DIR); - - // modpackFile is relative path to ~/.minecraft/ (content format) so if it starts with /automodpack/ we dont want it if (formattedFile.startsWith("/automodpack/")) { return null; } @@ -308,13 +297,12 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path type = "other"; } - String sha1 = SmartFileUtils.getHash(file); + String sha1 = metadataCache != null ? metadataCache.getOrComputeHash(file) : SmartFileUtils.getHash(file); // For CF API String murmur = null; if (type.equals("mod") || type.equals("shader") || type.equals("resourcepack")) { - // get murmur hash from previousContent.list of item with same sha1 - murmur = sha1MurmurMapPreviousContent.get(sha1); + murmur = sha1MurmurMapPreviousContent.get(sha1); // Get from cache if (murmur == null) { murmur = SmartFileUtils.getCurseforgeMurmurHash(file); } diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java index fdea654bb..7692b950b 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java @@ -1,10 +1,10 @@ package pl.skidam.automodpack_core.modpack; import pl.skidam.automodpack_core.utils.*; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.util.*; import java.util.concurrent.*; @@ -13,6 +13,11 @@ public class ModpackExecutor { private final ThreadPoolExecutor CREATION_EXECUTOR = (ThreadPoolExecutor) Executors.newFixedThreadPool(Math.max(1, Runtime.getRuntime().availableProcessors() * 2), new CustomThreadFactoryBuilder().setNameFormat("AutoModpackCreation-%d").build()); public final Map modpacks = new ConcurrentHashMap<>(); + private final FileMetadataCache fileMetadataCache; + + public ModpackExecutor() { + this.fileMetadataCache = new FileMetadataCache(hashCacheDBFile); + } private ModpackContent init() { if (isGenerating()) { @@ -29,11 +34,11 @@ private ModpackContent init() { Files.createDirectory(hostContentModpackDir.resolve("resourcepacks")); } } catch (IOException e) { - e.printStackTrace(); + LOGGER.error("Failed to create modpack content directory!", e); + return null; } - Path cwd = Path.of(System.getProperty("user.dir")); - return new ModpackContent(serverConfig.modpackName, cwd, hostContentModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, CREATION_EXECUTOR); + return new ModpackContent(serverConfig.modpackName, SmartFileUtils.CWD, hostContentModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, CREATION_EXECUTOR, fileMetadataCache); } public boolean generateNew(ModpackContent content) { @@ -71,5 +76,6 @@ public ThreadPoolExecutor getExecutor() { public void stop() { CREATION_EXECUTOR.shutdown(); + fileMetadataCache.close(); } } \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java index 095279b3c..2ed40f1d9 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java @@ -37,7 +37,7 @@ public class NettyServer { public static final AttributeKey CHUNK_SIZE = AttributeKey.valueOf("CHUNK_SIZE"); public static final AttributeKey PROTOCOL_VERSION = AttributeKey.valueOf("PROTOCOL_VERSION"); private final Map connections = new ConcurrentHashMap<>(); - private final Map paths = new ConcurrentHashMap<>(); + private Map paths = new ConcurrentHashMap<>(); private MultithreadEventLoopGroup eventLoopGroup; private ChannelFuture serverChannel; private Boolean shouldHost = false; // needed for stop modpack hosting for minecraft port @@ -64,8 +64,8 @@ public String getCertificateFingerprint() { return certificateFingerprint; } - public void addPaths(ObservableMap paths) { - this.paths.putAll(paths.getMap()); + public void setPaths(ObservableMap paths) { + this.paths = paths.getMap(); paths.addOnPutCallback(this.paths::put); paths.addOnRemoveCallback(this.paths::remove); } diff --git a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java index 2b3ab7603..8c41b90d5 100644 --- a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java +++ b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java @@ -99,7 +99,7 @@ void modpackTest() { Constants.serverConfig = new Jsons.ServerConfigFieldsV2(); Constants.serverConfig.autoExcludeUnnecessaryFiles = false; - ModpackContent content = new ModpackContent("TestPack", null, testFilesDir, new HashSet<>(), new HashSet<>(editable), new HashSet<>(), new ModpackExecutor().getExecutor()); + ModpackContent content = new ModpackContent("TestPack", null, testFilesDir, new HashSet<>(), new HashSet<>(editable), new HashSet<>(), new ModpackExecutor().getExecutor(), null); content.create(); boolean correct = true; From ac0d01a6777ede43dbe0ceb2a48d52384f691a5c Mon Sep 17 00:00:00 2001 From: skidam Date: Sat, 24 Jan 2026 20:48:17 +0100 Subject: [PATCH 03/22] Refactor FileInspection to use FileMetadataCache for hash retrieval --- .../pl/skidam/automodpack_core/modpack/ModpackContent.java | 2 +- .../pl/skidam/automodpack_core/utils/FileInspection.java | 5 +++-- .../automodpack_loader_core/client/ModpackUpdater.java | 4 ++-- .../skidam/automodpack_loader_core/client/ModpackUtils.java | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java index 7a144fef2..56bbfff36 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java @@ -297,7 +297,7 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path type = "other"; } - String sha1 = metadataCache != null ? metadataCache.getOrComputeHash(file) : SmartFileUtils.getHash(file); + String sha1 = metadataCache != null ? metadataCache.getHashOrNull(file) : SmartFileUtils.getHash(file); // For CF API String murmur = null; diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java index f42053f57..121feec65 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java @@ -9,6 +9,7 @@ import org.tomlj.TomlTable; import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.loader.LoaderManagerService; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; import java.io.BufferedReader; import java.io.IOException; @@ -45,11 +46,11 @@ public static boolean isMod(Path file) { public record Mod(String modID, String hash, Collection providesIDs, String modVersion, Path modPath, LoaderManagerService.EnvironmentType environmentType, Collection dependencies) {} public record HashPathPair(String hash, Path path) { } - public static Mod getMod(Path file) { + public static Mod getMod(Path file, FileMetadataCache cache) { if (!Files.isRegularFile(file)) return null; if (!file.getFileName().toString().endsWith(".jar")) return null; - String hash = SmartFileUtils.getHash(file); + String hash = cache != null ? cache.getHashOrNull(file) : SmartFileUtils.getHash(file); if (hash == null) { LOGGER.error("Failed to get hash for file: {}", file); return null; diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index ae3a15d3b..0c810972e 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -450,7 +450,7 @@ private boolean applyModpack(FileMetadataCache cache) throws Exception { try (Stream stream = Files.list(modpackModsDir)) { stream.forEach(path -> { modpackMods.add(path); - FileInspection.Mod mod = FileInspection.getMod(path); + FileInspection.Mod mod = FileInspection.getMod(path, cache); if (mod != null) { modpackModList.add(mod); } @@ -463,7 +463,7 @@ private boolean applyModpack(FileMetadataCache cache) throws Exception { if (Files.exists(standardModsDir)) { try (Stream stream = Files.list(standardModsDir)) { stream.forEach(path -> { - FileInspection.Mod mod = FileInspection.getMod(path); + FileInspection.Mod mod = FileInspection.getMod(path, cache); if (mod != null) { standardModList.add(mod); } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java index cdb6ac24a..245846c8c 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java @@ -316,7 +316,7 @@ public static boolean fixNestedMods(List conflictingNestedMo needsRestart = true; LOGGER.info("Copying nested mod {} to standard mods folder", standardModPath.getFileName()); SmartFileUtils.copyFile(modPath, standardModPath); - var newMod = FileInspection.getMod(standardModPath); + var newMod = FileInspection.getMod(standardModPath, cache); if (newMod != null) standardModList.add(newMod); // important } } From f43f1adcc2c739223cbf36ecfa31fe225c77e074 Mon Sep 17 00:00:00 2001 From: skidam Date: Sat, 24 Jan 2026 21:15:22 +0100 Subject: [PATCH 04/22] Update ModpackContentItems to include forceCopy field and enhance logging during content generation --- .../skidam/automodpack_core/config/Jsons.java | 2 +- .../modpack/ModpackContent.java | 6 +++- .../automodpack_core/modpack/ModpackTest.java | 36 +++++++++---------- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/config/Jsons.java b/core/src/main/java/pl/skidam/automodpack_core/config/Jsons.java index 39ea06c19..09867ce4b 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/config/Jsons.java +++ b/core/src/main/java/pl/skidam/automodpack_core/config/Jsons.java @@ -180,7 +180,7 @@ public ModpackContentItem(String file, String size, String type, boolean editabl @Override public String toString() { - return String.format("ModpackContentItems(file=%s, size=%s, type=%s, editable=%s, sha1=%s, murmur=%s)", file, size, type, editable, sha1, murmur); + return String.format("ModpackContentItems(file=%s, size=%s, type=%s, editable=%s, forceCopy=%s, sha1=%s, murmur=%s)", file, size, type, editable, forceCopy, sha1, murmur); } // if the relative file path is the same, we consider the items equal diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java index 56bbfff36..b55fdc5c8 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java @@ -94,7 +94,9 @@ public boolean create() { List> futures = filesToProcess.entrySet().stream() .map(entry -> CompletableFuture.supplyAsync(() -> { try { - return generateContent(entry.getValue(), entry.getKey()); + var contentEntry = generateContent(entry.getValue(), entry.getKey()); + LOGGER.debug("Generated modpack content for {}", entry.getValue()); + return contentEntry; } catch (Exception e) { LOGGER.error("Error generating content for {}", entry.getValue(), e); return null; @@ -115,6 +117,8 @@ public boolean create() { if (list.isEmpty()) { LOGGER.warn("Modpack is empty!"); return false; + } else { + LOGGER.info("Modpack generated with {} files!", list.size()); } } catch (Exception e) { LOGGER.error("Error while generating modpack!", e); diff --git a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java index 8c41b90d5..0eebc74e0 100644 --- a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java +++ b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java @@ -76,24 +76,24 @@ void modpackTest() { editable.forEach(System.out::println); var correctResults = List.of( - "ModpackContentItems(file=/shaders/notashader.zip, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/config/config-mod.json5, size=1, type=config, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/random directory/random-config.yaml, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/file.txt, size=1, type=other, editable=true, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/shaders/shader1.zip, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/config/config.json, size=1, type=config, editable=true, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/client-mod-1.19.jar, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/shaders/shader2.zip, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/config/mod-config.toml, size=1, type=config, editable=true, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/shaders/shader3.zip, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/client-mod-1.20.jar, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/shaders/shaderconfig.txt, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/mod, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/config/random-options.txt, size=1, type=config, editable=true, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/mod-1.19.jar, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/mod-1.20.jar, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/server-mod-1.19.jar, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", - "ModpackContentItems(file=/mods/server-mod-1.20.jar, size=1, type=other, editable=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)" + "ModpackContentItems(file=/shaders/notashader.zip, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/config/config-mod.json5, size=1, type=config, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/random directory/random-config.yaml, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/file.txt, size=1, type=other, editable=true, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/shaders/shader1.zip, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/config/config.json, size=1, type=config, editable=true, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/client-mod-1.19.jar, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/shaders/shader2.zip, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/config/mod-config.toml, size=1, type=config, editable=true, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/shaders/shader3.zip, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/client-mod-1.20.jar, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/shaders/shaderconfig.txt, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/mod, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/config/random-options.txt, size=1, type=config, editable=true, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/mod-1.19.jar, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/mod-1.20.jar, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/server-mod-1.19.jar, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)", + "ModpackContentItems(file=/mods/server-mod-1.20.jar, size=1, type=other, editable=false, forceCopy=false, sha1=86f7e437faa5a7fce15d1ddcb9eaeaea377667b8, murmur=null)" ); Constants.serverConfig = new Jsons.ServerConfigFieldsV2(); From 5010029cbfdae5374a2aa0872a02b5fe7618fb1f Mon Sep 17 00:00:00 2001 From: skidam Date: Sat, 24 Jan 2026 21:23:12 +0100 Subject: [PATCH 05/22] Update fabric-loom plugin version to 1.15-SNAPSHOT --- stonecutter.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stonecutter.gradle.kts b/stonecutter.gradle.kts index 769163e1b..af7947066 100644 --- a/stonecutter.gradle.kts +++ b/stonecutter.gradle.kts @@ -1,7 +1,7 @@ plugins { id("dev.kikugie.stonecutter") kotlin("jvm") version "2.3.0" apply false - id("fabric-loom") version "1.14-SNAPSHOT" apply false + id("fabric-loom") version "1.15-SNAPSHOT" apply false id("net.neoforged.moddev") version "2.0.139" apply false id("com.gradleup.shadow") version "9.3.0" apply false id("org.moddedmc.wiki.toolkit") version "0.4+" From 94bb7250f32781f1bf3b29e1255fe39a5b1df1c5 Mon Sep 17 00:00:00 2001 From: skidam Date: Sun, 25 Jan 2026 00:43:09 +0100 Subject: [PATCH 06/22] Try revert to putAll instead of overwrite --- .../skidam/automodpack_core/protocol/netty/NettyServer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java index 2ed40f1d9..e022cf9e2 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/NettyServer.java @@ -37,7 +37,7 @@ public class NettyServer { public static final AttributeKey CHUNK_SIZE = AttributeKey.valueOf("CHUNK_SIZE"); public static final AttributeKey PROTOCOL_VERSION = AttributeKey.valueOf("PROTOCOL_VERSION"); private final Map connections = new ConcurrentHashMap<>(); - private Map paths = new ConcurrentHashMap<>(); + private final Map paths = new ConcurrentHashMap<>(); private MultithreadEventLoopGroup eventLoopGroup; private ChannelFuture serverChannel; private Boolean shouldHost = false; // needed for stop modpack hosting for minecraft port @@ -65,7 +65,7 @@ public String getCertificateFingerprint() { } public void setPaths(ObservableMap paths) { - this.paths = paths.getMap(); + this.paths.putAll(paths.getMap()); paths.addOnPutCallback(this.paths::put); paths.addOnRemoveCallback(this.paths::remove); } From 29e3f2ffb7bdecf2ba567ebfd9bb816651c9ea84 Mon Sep 17 00:00:00 2001 From: skidam Date: Sun, 25 Jan 2026 09:38:45 +0100 Subject: [PATCH 07/22] Refactor FileMetadataCache to support shared cache instances and improve resource management --- .../pl/skidam/automodpack_core/Server.java | 2 +- .../modpack/ModpackContent.java | 21 +++++----- .../modpack/ModpackExecutor.java | 20 +++++----- .../netty/handler/ServerMessageHandler.java | 35 ++++++++-------- .../utils/cache/FileMetadataCache.java | 40 ++++++++++++++++--- .../automodpack_core/modpack/ModpackTest.java | 4 +- .../client/ModpackUpdater.java | 6 +-- .../client/ModpackUtils.java | 2 +- 8 files changed, 80 insertions(+), 50 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/Server.java b/core/src/main/java/pl/skidam/automodpack_core/Server.java index a843f1731..a1993b433 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/Server.java +++ b/core/src/main/java/pl/skidam/automodpack_core/Server.java @@ -60,7 +60,7 @@ public static void main(String[] args) { mainModpackDir.toFile().mkdirs(); ModpackExecutor modpackExecutor = new ModpackExecutor(); - ModpackContent modpackContent = new ModpackContent(serverConfig.modpackName, null, mainModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, modpackExecutor.getExecutor(), null); + ModpackContent modpackContent = new ModpackContent(serverConfig.modpackName, null, mainModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, modpackExecutor.getExecutor()); boolean generated = modpackExecutor.generateNew(modpackContent); if (generated) { diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java index b55fdc5c8..2a4e2b678 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java @@ -26,14 +26,12 @@ public class ModpackContent { private final FileTreeScanner FORCE_COPY_FILES_TO_STANDARD_LOCATION; private final Path MODPACK_DIR; private final ThreadPoolExecutor CREATION_EXECUTOR; - private final FileMetadataCache metadataCache; private final Map sha1MurmurMapPreviousContent = new HashMap<>(); - public ModpackContent(String modpackName, Path cwd, Path modpackDir, Set syncedFiles, Set allowEditsInFiles, Set forceCopyFilesToStandardLocation, ThreadPoolExecutor CREATION_EXECUTOR, FileMetadataCache metadataCache) { + public ModpackContent(String modpackName, Path cwd, Path modpackDir, Set syncedFiles, Set allowEditsInFiles, Set forceCopyFilesToStandardLocation, ThreadPoolExecutor CREATION_EXECUTOR) { this.MODPACK_NAME = modpackName; this.MODPACK_DIR = modpackDir; this.CREATION_EXECUTOR = CREATION_EXECUTOR; - this.metadataCache = metadataCache; Set directoriesToSearch = new HashSet<>(2); if (MODPACK_DIR != null) directoriesToSearch.add(MODPACK_DIR); if (cwd != null) { @@ -50,7 +48,7 @@ public String getModpackName() { return MODPACK_NAME; } - public boolean create() { + public boolean create(FileMetadataCache cache) { Set computedFilesToDelete = new HashSet<>(); try { @@ -94,7 +92,7 @@ public boolean create() { List> futures = filesToProcess.entrySet().stream() .map(entry -> CompletableFuture.supplyAsync(() -> { try { - var contentEntry = generateContent(entry.getValue(), entry.getKey()); + var contentEntry = generateContent(entry.getValue(), entry.getKey(), cache); LOGGER.debug("Generated modpack content for {}", entry.getValue()); return contentEntry; } catch (Exception e) { @@ -186,14 +184,15 @@ public synchronized void saveModpackContent(Set replaceAsync(Path file) { - return CompletableFuture.runAsync(() -> replace(file), CREATION_EXECUTOR); + public CompletableFuture replaceAsync(Path file, FileMetadataCache cache) { + return CompletableFuture.runAsync(() -> replace(file, cache), CREATION_EXECUTOR); } - public void replace(Path file) { + public void replace(Path file, FileMetadataCache cache) { remove(file); try { - Jsons.ModpackContentFields.ModpackContentItem item = generateContent(file, SmartFileUtils.formatPath(file, MODPACK_DIR)); + String modpackFile = SmartFileUtils.formatPath(file, MODPACK_DIR); + Jsons.ModpackContentFields.ModpackContentItem item = generateContent(file, modpackFile, cache); if (item != null) { LOGGER.info("generated content for {}", item.file); synchronized (list) { @@ -231,7 +230,7 @@ public static boolean isInnerFile(Path file) { return isInner; } - private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path file, final String formattedFile) throws Exception { + private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path file, final String formattedFile, FileMetadataCache cache) throws Exception { if (!Files.isRegularFile(file)) return null; if (serverConfig == null) { @@ -301,7 +300,7 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path type = "other"; } - String sha1 = metadataCache != null ? metadataCache.getHashOrNull(file) : SmartFileUtils.getHash(file); + String sha1 = cache != null ? cache.getHashOrNull(file) : SmartFileUtils.getHash(file); // For CF API String murmur = null; diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java index 7692b950b..3535c9142 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackExecutor.java @@ -13,15 +13,10 @@ public class ModpackExecutor { private final ThreadPoolExecutor CREATION_EXECUTOR = (ThreadPoolExecutor) Executors.newFixedThreadPool(Math.max(1, Runtime.getRuntime().availableProcessors() * 2), new CustomThreadFactoryBuilder().setNameFormat("AutoModpackCreation-%d").build()); public final Map modpacks = new ConcurrentHashMap<>(); - private final FileMetadataCache fileMetadataCache; - - public ModpackExecutor() { - this.fileMetadataCache = new FileMetadataCache(hashCacheDBFile); - } private ModpackContent init() { if (isGenerating()) { - LOGGER.error("Called generate() twice!"); + LOGGER.error("Called init() while generating!"); return null; } @@ -38,12 +33,15 @@ private ModpackContent init() { return null; } - return new ModpackContent(serverConfig.modpackName, SmartFileUtils.CWD, hostContentModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, CREATION_EXECUTOR, fileMetadataCache); + return new ModpackContent(serverConfig.modpackName, SmartFileUtils.CWD, hostContentModpackDir, serverConfig.syncedFiles, serverConfig.allowEditsInFiles, serverConfig.forceCopyFilesToStandardLocation, CREATION_EXECUTOR); } public boolean generateNew(ModpackContent content) { if (content == null) return false; - boolean generated = content.create(); + boolean generated; + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { + generated = content.create(cache); + } modpacks.put(content.getModpackName(), content); return generated; } @@ -51,7 +49,10 @@ public boolean generateNew(ModpackContent content) { public boolean generateNew() { ModpackContent content = init(); if (content == null) return false; - boolean generated = content.create(); + boolean generated; + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { + generated = content.create(cache); + } modpacks.put(content.getModpackName(), content); return generated; } @@ -76,6 +77,5 @@ public ThreadPoolExecutor getExecutor() { public void stop() { CREATION_EXECUTOR.shutdown(); - fileMetadataCache.close(); } } \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java index 97796904a..3cf4e80a4 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/netty/handler/ServerMessageHandler.java @@ -25,6 +25,7 @@ import pl.skidam.automodpack_core.protocol.netty.message.request.FileRequestMessage; import pl.skidam.automodpack_core.protocol.netty.message.request.RefreshRequestMessage; import pl.skidam.automodpack_core.utils.LockFreeInputStream; +import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; public class ServerMessageHandler extends SimpleChannelInboundHandler { @@ -85,27 +86,29 @@ private void refreshModpackFiles(ChannelHandlerContext context, byte[][] FileHas List> creationFutures = new ArrayList<>(); Set modpacks = new HashSet<>(); - for (String hash : hashes) { - final Optional optionalPath = resolvePath(hash); - if (optionalPath.isEmpty()) continue; - Path path = optionalPath.get(); - ModpackContent modpack = null; + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { + for (String hash : hashes) { + final Optional optionalPath = resolvePath(hash); + if (optionalPath.isEmpty()) continue; + Path path = optionalPath.get(); + ModpackContent modpack = null; - for (var content : modpackExecutor.modpacks.values()) { - if (!content.pathsMap.getMap().containsKey(hash)) { - continue; + for (var content : modpackExecutor.modpacks.values()) { + if (!content.pathsMap.getMap().containsKey(hash)) { + continue; + } + + modpack = content; + break; } - modpack = content; - break; - } + if (modpack == null) { + continue; + } - if (modpack == null) { - continue; + modpacks.add(modpack); + creationFutures.add(modpack.replaceAsync(path, cache)); } - - modpacks.add(modpack); - creationFutures.add(modpack.replaceAsync(path)); } creationFutures.forEach(CompletableFuture::join); diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java index 32a9ed4d5..43a13e5d3 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java @@ -9,17 +9,37 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import static pl.skidam.automodpack_core.Constants.LOGGER; public class FileMetadataCache implements AutoCloseable { + private static final Map INSTANCES = new HashMap<>(); + private final AtomicInteger refCount = new AtomicInteger(1); + private final Path dbPath; + + public static synchronized FileMetadataCache open(Path path) { + Path absPath = path.toAbsolutePath().normalize(); + FileMetadataCache existing = INSTANCES.get(absPath); + + if (existing != null) { + existing.refCount.incrementAndGet(); + return existing; + } + + FileMetadataCache newCache = new FileMetadataCache(absPath); + INSTANCES.put(absPath, newCache); + return newCache; + } + + private final MVStore store; private final MVMap fileMetadataMap; private final AtomicInteger uncommittedWrites = new AtomicInteger(0); private static final int COMMIT_THRESHOLD = 50; - private final Object[] locks = new Object[64]; public record CachedFile( @@ -32,7 +52,8 @@ public record CachedFile( private static final long serialVersionUID = 1L; } - public FileMetadataCache(Path dbPath) { + private FileMetadataCache(Path dbPath) { + this.dbPath = dbPath; this.store = new MVStore.Builder() .fileName(dbPath.toString()) .cacheSize(20) @@ -130,12 +151,19 @@ private void checkAndCommit() { } } - @Override public void close() { - if (!store.isClosed()) { - store.commit(); - store.close(); + synchronized (FileMetadataCache.class) { + if (refCount.decrementAndGet() <= 0) { + try { + if (!store.isClosed()) { + store.commit(); + store.close(); + } + } finally { + INSTANCES.remove(this.dbPath, this); + } + } } } } \ No newline at end of file diff --git a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java index 0eebc74e0..1ee4cfa5b 100644 --- a/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java +++ b/core/src/test/java/pl/skidam/automodpack_core/modpack/ModpackTest.java @@ -99,8 +99,8 @@ void modpackTest() { Constants.serverConfig = new Jsons.ServerConfigFieldsV2(); Constants.serverConfig.autoExcludeUnnecessaryFiles = false; - ModpackContent content = new ModpackContent("TestPack", null, testFilesDir, new HashSet<>(), new HashSet<>(editable), new HashSet<>(), new ModpackExecutor().getExecutor(), null); - content.create(); + ModpackContent content = new ModpackContent("TestPack", null, testFilesDir, new HashSet<>(), new HashSet<>(editable), new HashSet<>(), new ModpackExecutor().getExecutor()); + content.create(null); boolean correct = true; diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index 0c810972e..a609355f9 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -64,7 +64,7 @@ public void processModpackUpdate(ModpackUtils.UpdateCheckResult result) { // Handle the case where serverModpackContent is null if (serverModpackContent == null) { - try (var cache = new FileMetadataCache(hashCacheDBFile)) { + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { CheckAndLoadModpack(cache); } return; @@ -101,7 +101,7 @@ public void processModpackUpdate(ModpackUtils.UpdateCheckResult result) { startUpdate(result.filesToUpdate()); } else { Files.writeString(modpackContentFile, serverModpackContentJson); - try (var cache = new FileMetadataCache(hashCacheDBFile)) { + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { CheckAndLoadModpack(cache); } } @@ -177,7 +177,7 @@ public void startUpdate(Set files long start = System.currentTimeMillis(); - try (var cache = new FileMetadataCache(hashCacheDBFile)) { + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { // Don't download files which already exist // The existing files will be copied over in the applyModpack method var finalFilesToUpdate = ModpackUtils.getOnlyNonExistingFiles(filesToUpdate, cache); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java index 245846c8c..4389846ac 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java @@ -64,7 +64,7 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac Set filesToUpdate = ConcurrentHashMap.newKeySet(); - try (var cache = new FileMetadataCache(hashCacheDBFile)) { + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { serverModpackContent.list.forEach(serverItem -> { Path serverItemPath = SmartFileUtils.getPath(modpackDir, serverItem.file); if (!existingFileTree.contains(serverItemPath)) { From af07caeb8f5af9895bc61be7cf6f6a7d445438c7 Mon Sep 17 00:00:00 2001 From: skidam Date: Sun, 25 Jan 2026 10:03:59 +0100 Subject: [PATCH 08/22] Cleanup FileMetadataCache --- .../utils/cache/FileMetadataCache.java | 64 ++++++++----------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java index 43a13e5d3..0c45aafb4 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java @@ -18,38 +18,32 @@ public class FileMetadataCache implements AutoCloseable { private static final Map INSTANCES = new HashMap<>(); - private final AtomicInteger refCount = new AtomicInteger(1); - private final Path dbPath; + private static final Object GLOBAL_LOCK = new Object(); - public static synchronized FileMetadataCache open(Path path) { - Path absPath = path.toAbsolutePath().normalize(); - FileMetadataCache existing = INSTANCES.get(absPath); + private final Path dbPath; + private final MVStore store; + private final MVMap fileMetadataMap; + private final AtomicInteger refCount = new AtomicInteger(1); - if (existing != null) { - existing.refCount.incrementAndGet(); - return existing; - } + private final Object[] locks = new Object[64]; - FileMetadataCache newCache = new FileMetadataCache(absPath); - INSTANCES.put(absPath, newCache); - return newCache; + public record CachedFile(String contentHash, long lastModified, long size, String fileKey) implements Serializable { + @java.io.Serial private static final long serialVersionUID = 1L; } + public static FileMetadataCache open(Path path) { + Path absPath = path.toAbsolutePath().normalize(); + synchronized (GLOBAL_LOCK) { + FileMetadataCache existing = INSTANCES.get(absPath); + if (existing != null) { + existing.refCount.incrementAndGet(); + return existing; + } - private final MVStore store; - private final MVMap fileMetadataMap; - private final AtomicInteger uncommittedWrites = new AtomicInteger(0); - private static final int COMMIT_THRESHOLD = 50; - private final Object[] locks = new Object[64]; - - public record CachedFile( - String contentHash, - long lastModified, - long size, - String fileKey - ) implements Serializable { - @java.io.Serial - private static final long serialVersionUID = 1L; + FileMetadataCache newCache = new FileMetadataCache(absPath); + INSTANCES.put(absPath, newCache); + return newCache; + } } private FileMetadataCache(Path dbPath) { @@ -57,7 +51,6 @@ private FileMetadataCache(Path dbPath) { this.store = new MVStore.Builder() .fileName(dbPath.toString()) .cacheSize(20) - .autoCommitDisabled() .open(); this.fileMetadataMap = store.openMap("file_metadata"); @@ -96,17 +89,13 @@ public String getOrComputeHash(Path file) throws IOException { CachedFile newRecord = new CachedFile(newHash, currentTime, currentSize, currentFileKey); fileMetadataMap.put(pathKey, newRecord); - checkAndCommit(); return newHash; } } private boolean isCacheValid(CachedFile cached, long size, long time, String key) { - if (cached == null) return false; - return cached.size() == size && - cached.lastModified() == time && - cached.fileKey().equals(key); + return cached != null && cached.size() == size && cached.lastModified() == time && cached.fileKey().equals(key); } public String getHashOrNull(Path path) { @@ -141,19 +130,20 @@ public void overwriteCache(Path file, String hash) throws IOException { CachedFile newRecord = new CachedFile(hash, currentTime, currentSize, currentFileKey); fileMetadataMap.put(pathKey, newRecord); - checkAndCommit(); } - private void checkAndCommit() { - if (uncommittedWrites.incrementAndGet() >= COMMIT_THRESHOLD) { - uncommittedWrites.set(0); + // TODO: Consider running periodically + public void cleanup() { + synchronized (store) { + fileMetadataMap.keySet().removeIf(pathString -> Files.notExists(Path.of(pathString))); store.commit(); + store.compactFile(2000); } } @Override public void close() { - synchronized (FileMetadataCache.class) { + synchronized (GLOBAL_LOCK) { if (refCount.decrementAndGet() <= 0) { try { if (!store.isClosed()) { From a76e27bd96383374e44557849899247cb12351ff Mon Sep 17 00:00:00 2001 From: skidam Date: Sun, 25 Jan 2026 16:05:58 +0100 Subject: [PATCH 09/22] Add ModFileCache reduces boot time by 40% --- core/build.gradle.kts | 1 + .../pl/skidam/automodpack_core/Constants.java | 1 + .../utils/FileInspection.java | 597 +++++++----------- .../utils/WorkaroundUtil.java | 13 +- .../utils/cache/ModFileCache.java | 122 ++++ .../client/ModpackUpdater.java | 68 +- .../client/ModpackUtils.java | 38 +- .../mods/ModpackLoader15.java | 18 +- .../mods/ModpackLoader16.java | 18 +- 9 files changed, 458 insertions(+), 418 deletions(-) create mode 100644 core/src/main/java/pl/skidam/automodpack_core/utils/cache/ModFileCache.java diff --git a/core/build.gradle.kts b/core/build.gradle.kts index a1ee153f8..352736652 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -28,6 +28,7 @@ val deps = listOf( dependencies { // minecraft/loaders uses these, so we cant just implement them because it wont resolve in gradle deps.forEach { compileOnly(it) } + deps.forEach { runtimeOnly(it) } deps.forEach { testImplementation(it) } testImplementation("org.junit.jupiter:junit-jupiter:6.0.1") diff --git a/core/src/main/java/pl/skidam/automodpack_core/Constants.java b/core/src/main/java/pl/skidam/automodpack_core/Constants.java index c61d03771..1bd9ee0e1 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/Constants.java +++ b/core/src/main/java/pl/skidam/automodpack_core/Constants.java @@ -42,6 +42,7 @@ public class Constants { public static Path serverConfigFile = automodpackDir.resolve("automodpack-server.json"); public static Path clientLocalMetadataFile = automodpackDir.resolve("automodpack-client-metadata.json"); public static Path hashCacheDBFile = automodpackDir.resolve("hash-cache.db"); + public static Path modCacheDBFile = automodpackDir.resolve("mod-cache.db"); public static Path clientDummyFilesFile = automodpackDir.resolve("automodpack-dummy-files.json"); public static Path clientDeletionTimeStamps = automodpackDir.resolve("automodpack-deletion-timestamps-files.json"); public static Path serverCoreConfigFile = automodpackDir.resolve("automodpack-core.json"); diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java index 121feec65..b3806d29e 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java @@ -11,14 +11,8 @@ import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; +import java.io.*; +import java.nio.file.*; import java.util.*; import java.util.stream.Stream; import java.util.zip.ZipEntry; @@ -31,24 +25,34 @@ public class FileInspection { private static final Gson GSON = new Gson(); private static final String LOADER = Constants.LOADER; - public static boolean isMod(Path file) { - if (!file.getFileName().toString().endsWith(".jar") || !Files.exists(file)) { - return false; + public record HashPathPair(String hash, Path path) {} + + public record Mod(Set IDs, String hash, String version, Path path, Set deps, Set nestedMods) implements Serializable { + + // Magic to de/serialize Path properly + + @Serial + private Object writeReplace() { + return new SerializationProxy(this); } - try (FileSystem fs = FileSystems.newFileSystem(file)) { - return getModID(fs) != null || hasSpecificServices(fs); - } catch (IOException e) { - return false; + private record SerializationProxy(Set IDs, String hash, String version, String pathString, Set deps, Set nestedMods) implements Serializable { + + public SerializationProxy(Mod mod) { + this(mod.IDs(), mod.hash(), mod.version(), mod.path() == null ? null : mod.path().toAbsolutePath().normalize().toString(), mod.deps(), mod.nestedMods()); + } + + @Serial + private Object readResolve() { + return new Mod(IDs, hash, version, pathString == null ? null : Path.of(pathString), deps, nestedMods); + } } } - public record Mod(String modID, String hash, Collection providesIDs, String modVersion, Path modPath, LoaderManagerService.EnvironmentType environmentType, Collection dependencies) {} - public record HashPathPair(String hash, Path path) { } + private record ModMetadata(String modId, String version, Set provides, Set deps, LoaderManagerService.EnvironmentType environment) {} public static Mod getMod(Path file, FileMetadataCache cache) { - if (!Files.isRegularFile(file)) return null; - if (!file.getFileName().toString().endsWith(".jar")) return null; + if (isJarInvalid(file)) return null; String hash = cache != null ? cache.getHashOrNull(file) : SmartFileUtils.getHash(file); if (hash == null) { @@ -56,421 +60,310 @@ public static Mod getMod(Path file, FileMetadataCache cache) { return null; } - // Open FS once for all metadata extractions try (FileSystem fs = FileSystems.newFileSystem(file)) { - String modId = (String) getModInfo(fs, "modId"); + ModMetadata meta = getModMetadata(fs); - if (modId != null) { - String modVersion = (String) getModInfo(fs, "version"); - LoaderManagerService.EnvironmentType environmentType = (LoaderManagerService.EnvironmentType) getModInfo(fs, "environment"); - Set dependencies = getModDependencies(fs); - Set providesIDs = getProvidedIDs(fs); + if (meta != null && meta.modId() != null) { + Set ids = new HashSet<>(meta.provides()); + ids.add(meta.modId()); - if (modVersion != null && dependencies != null) { - return new Mod(modId, hash, providesIDs, modVersion, file, environmentType, dependencies); - } + Set nestedMods = scanForNestedMods(fs); - LOGGER.error("Not enough mod information for file: {} modId: {}, modVersion: {}, dependencies: {}", file, modId, modVersion, dependencies); + if (meta.version() != null) { + return new Mod(ids, hash, meta.version(), file, meta.deps(), nestedMods); + } + LOGGER.error("Incomplete mod info for file: {} (ID: {}, Ver: {})", file, meta.modId(), meta.version()); } } catch (IOException e) { - LOGGER.debug("Failed to get mod info for file: {}", file); + LOGGER.debug("Failed to inspect mod file: {}", file); } - return null; } - private static final Set services = Set.of( - "META-INF/services/net.minecraftforge.forgespi.locating.IModLocator", - "META-INF/services/net.minecraftforge.forgespi.locating.IDependencyLocator", - "META-INF/services/net.minecraftforge.forgespi.language.IModLanguageProvider", - "META-INF/services/net.neoforged.neoforgespi.locating.IModLocator", - "META-INF/services/net.neoforged.neoforgespi.locating.IDependencyLocator", - "META-INF/services/net.neoforged.neoforgespi.locating.IModLanguageLoader", - "META-INF/services/net.neoforged.neoforgespi.locating.IModFileCandidateLocator", - "META-INF/services/net.neoforged.neoforgespi.earlywindow.GraphicsBootstrapper" - ); - - // Checks for neo/forge mod locators - public static boolean isModCompatible(Path file) { - if (!file.getFileName().toString().endsWith(".jar") || !Files.exists(file)) { - return false; - } - + public static boolean isMod(Path file) { + if (isJarInvalid(file)) return false; try (FileSystem fs = FileSystems.newFileSystem(file)) { - String entryPathString = switch (LOADER) { - case "neoforge" -> "META-INF/neoforge.mods.toml"; - case "fabric" -> "fabric.mod.json"; - case "forge" -> "META-INF/mods.toml"; - case "quilt" -> "quilt.mod.json"; - default -> null; - }; - - if (entryPathString != null && Files.exists(fs.getPath(entryPathString))) { - return true; - } - - if ("forge".equals(LOADER) || "neoforge".equals(LOADER)) { - if (hasSpecificServices(fs)) { - return true; - } - } - + return getModMetadata(fs) != null || hasSpecificServices(fs); } catch (IOException e) { - // Ignore - } - - return false; - } - - public static boolean hasSpecificServices(Path file) { - if (!file.getFileName().toString().endsWith(".jar") || !Files.exists(file)) { return false; } - - try (FileSystem fs = FileSystems.newFileSystem(file)) { - return hasSpecificServices(fs); - } catch (IOException e) { - LOGGER.error("Error reading file {}: {}", file, e.getMessage()); - } - return false; } - public static boolean hasSpecificServices(FileSystem fs) { - // Direct Service Lookup (Fast) - for (String service : services) { - if (Files.exists(fs.getPath(service))) { - return true; - } - } - - // Jar-in-Jar Scan (Slower) - Path jarJarDir = fs.getPath("META-INF", "jarjar"); - if (!Files.exists(jarJarDir)) { - return false; - } - - try (Stream walk = Files.walk(jarJarDir, 1)) { - for (Path nestedJarPath : walk.toList()) { - // Skip non-jar entries - if (nestedJarPath.equals(jarJarDir) || !nestedJarPath.toString().endsWith(".jar")) { - continue; - } + public static boolean isModCompatible(Path file) { + if (isJarInvalid(file)) return false; - // Optimization: Use Files.newInputStream directly for nested zip entries - try (InputStream inputStream = Files.newInputStream(nestedJarPath); - ZipInputStream zipInputStream = new ZipInputStream(inputStream)) { + try (FileSystem fs = FileSystems.newFileSystem(file)) { + Path metaPath = getMetadataPath(fs); + if (metaPath != null) return true; - ZipEntry nestedEntry; - while ((nestedEntry = zipInputStream.getNextEntry()) != null) { - if (services.contains(nestedEntry.getName())) { - return true; - } - } - } catch (IOException e) { - LOGGER.error("Error reading nested JAR in {}: {}", nestedJarPath, e.getMessage()); - } + if ("forge".equals(LOADER) || "neoforge".equals(LOADER)) { + return hasSpecificServices(fs); } } catch (IOException e) { - LOGGER.error("Error examining JarJar in {}", fs, e); + LOGGER.error("Error examining JarJar in {}", e); } - return false; } - public static Path getMetadataPath(FileSystem fs) { - String preferredEntry = switch (LOADER) { - case "neoforge" -> "META-INF/neoforge.mods.toml"; - case "fabric" -> "fabric.mod.json"; - case "forge" -> "META-INF/mods.toml"; - case "quilt" -> "quilt.mod.json"; - default -> null; - }; - - if (preferredEntry != null) { - Path path = fs.getPath(preferredEntry); - if (Files.exists(path)) return path; - } - - String[] fallbackEntries = { - "META-INF/neoforge.mods.toml", - "fabric.mod.json", - "META-INF/mods.toml", - "quilt.mod.json" - }; - - for (String entryName : fallbackEntries) { - if (entryName.equals(preferredEntry)) continue; - - Path path = fs.getPath(entryName); - if (Files.exists(path)) return path; - } - - return null; - } - public static String getModVersion(Path file) { - return (String) getModInfo(file, "version"); + return extractBasicInfo(file, ModMetadata::version); } public static String getModID(Path file) { - return (String) getModInfo(file, "modId"); + return extractBasicInfo(file, ModMetadata::modId); } public static LoaderManagerService.EnvironmentType getModEnvironment(Path file) { - return (LoaderManagerService.EnvironmentType) getModInfo(file, "environment"); + return extractBasicInfo(file, ModMetadata::environment); } - private static String getModID(FileSystem fs) { - return (String) getModInfo(fs, "modId"); + private static boolean isJarInvalid(Path file) { + return file == null || !Files.exists(file) || !file.getFileName().toString().endsWith(".jar"); } - @SuppressWarnings("unchecked") - private static Set getProvidedIDs(FileSystem fs) { - return (Set) getModInfo(fs, "provides"); + private static T extractBasicInfo(Path file, java.util.function.Function extractor) { + if (isJarInvalid(file)) return null; + try (FileSystem fs = FileSystems.newFileSystem(file)) { + ModMetadata meta = getModMetadata(fs); + return meta != null ? extractor.apply(meta) : null; + } catch (IOException e) { + LOGGER.error("Error reading mod file {}: {}", file, e.getMessage()); + } + return null; } - @SuppressWarnings("unchecked") - private static Set getModDependencies(FileSystem fs) { - return (Set) getModInfo(fs, "dependencies"); + // TODO optimize it by caching and scanning only defined paths + private static Set scanForNestedMods(FileSystem parentFs) { + Set nestedMods = new HashSet<>(); + try (Stream walk = Files.walk(parentFs.getPath("/"))) { + for (Path path : walk.toList()) { + if (path.toString().endsWith(".jar") && !path.equals(parentFs.getPath("/"))) { + try (InputStream is = Files.newInputStream(path)) { + Mod nested = readModFromStream(path, is); + if (nested != null) nestedMods.add(nested); + } catch (IOException e) { + LOGGER.debug("Skipping unreadable nested jar: {}", path); + } + } + } + } catch (IOException e) { + LOGGER.error("Error scanning nested mods: {}", e.getMessage()); + } + return nestedMods; } - private static boolean isBasicInfo(String infoType) { - return "version".equals(infoType) || "modId".equals(infoType) || "environment".equals(infoType); - } + /** + * Reads a JAR from an InputStream (recursively) without mounting it as a FileSystem. + */ + private static Mod readModFromStream(Path virtualPath, InputStream is) { + // ZipInputStream must NOT close the underlying stream if it's a child stream + ZipInputStream zis = new ZipInputStream(is); + ZipEntry entry; + ModMetadata metadata = null; + Set nestedChildren = new HashSet<>(); - private static Object getModInfo(Path file, String infoType) { - if (!file.getFileName().toString().endsWith(".jar") || !Files.exists(file)) { - return isBasicInfo(infoType) ? null : Set.of(); - } + try { + while ((entry = zis.getNextEntry()) != null) { + String name = entry.getName(); - try (FileSystem fs = FileSystems.newFileSystem(file)) { - return getModInfo(fs, infoType); + if (isMetadataFilename(name)) { + // Prevent reader from closing the ZipInputStream + BufferedReader reader = new BufferedReader(new InputStreamReader(new FilterInputStream(zis) { + @Override public void close() {} + })); + + if (name.endsWith(".toml")) metadata = parseTomlMetadata(reader); + else metadata = parseJsonMetadata(reader); + } + else if (name.endsWith(".jar")) { + // Wrap ZIS to protect current stream position + Mod child = readModFromStream(virtualPath.resolve(name), new FilterInputStream(zis) { + @Override public void close() {} + }); + if (child != null) nestedChildren.add(child); + } + } } catch (IOException e) { - LOGGER.error("Error reading mod file {}: {}", file, e.getMessage()); + LOGGER.debug("Error processing stream for {}", virtualPath); } - return isBasicInfo(infoType) ? null : Set.of(); - } - private static Object getModInfo(FileSystem fs, String infoType) { - Path metadataPath = getMetadataPath(fs); - - if (metadataPath == null || !Files.exists(metadataPath)) { - return isBasicInfo(infoType) ? null : Set.of(); + if (metadata != null && metadata.modId() != null) { + Set ids = new HashSet<>(metadata.provides()); + ids.add(metadata.modId()); + // Investigate if we need hash or not + return new Mod(ids, null, metadata.version(), virtualPath, metadata.deps(), nestedChildren); } + return null; + } - try (InputStream stream = Files.newInputStream(metadataPath); - BufferedReader reader = new BufferedReader(new InputStreamReader(stream))) { + private static ModMetadata getModMetadata(FileSystem fs) { + Path metaPath = getMetadataPath(fs); + if (metaPath == null) return null; - if (metadataPath.getFileName().toString().endsWith("mods.toml")) { - return getModInfoFromToml(reader, infoType); + try (BufferedReader reader = Files.newBufferedReader(metaPath)) { + if (metaPath.toString().endsWith(".toml")) { + return parseTomlMetadata(reader); } else { - return getModInfoFromJson(reader, infoType); + return parseJsonMetadata(reader); } } catch (IOException e) { - LOGGER.error("Error reading metadata {}: {}", metadataPath, e.getMessage()); + LOGGER.error("Error parsing metadata {}: {}", metaPath, e.getMessage()); } - - return isBasicInfo(infoType) ? null : Set.of(); + return null; } - private static Object getModInfoFromToml(BufferedReader reader, String infoType) { + private static ModMetadata parseTomlMetadata(BufferedReader reader) { try { TomlParseResult result = Toml.parse(reader); - result.errors().forEach(error -> LOGGER.error(error.toString())); - - TomlArray modsArray = result.getArray("mods"); - if (modsArray == null) { - return isBasicInfo(infoType) ? null : Set.of(); - } + TomlArray mods = result.getArray("mods"); + if (mods == null || mods.isEmpty()) return null; - switch (infoType) { - case "version" -> { - String modVersion = null; - for (Object o : modsArray.toList()) { - TomlTable mod = (TomlTable) o; - if (mod != null) { - modVersion = mod.getString("version"); - } - } - return modVersion != null ? modVersion : "1"; - } - case "modId" -> { - String modID = null; - for (Object o : modsArray.toList()) { - TomlTable mod = (TomlTable) o; - if (mod != null) { - modID = mod.getString("modId"); - } - } - return modID; - } - case "provides" -> { - Set providedIDs = new HashSet<>(); - for (Object o : modsArray.toList()) { - TomlTable mod = (TomlTable) o; - if (mod != null) { - TomlArray providesArray = mod.getArray("provides"); - if (providesArray != null) { - for (int j = 0; j < providesArray.size(); j++) { - String id = providesArray.getString(j); - if (id != null && !id.isEmpty()) { - providedIDs.add(id); - } - } - } - } - } - return providedIDs; - } - case "dependencies" -> { - Set dependencies = new HashSet<>(); - - String modID = null; - for (Object o : modsArray.toList()) { - TomlTable mod = (TomlTable) o; - if (mod != null) { - modID = mod.getString("modId"); - } - } + String modId = null; + String version = "1"; + Set provides = new HashSet<>(); + Set deps = new HashSet<>(); + LoaderManagerService.EnvironmentType env = LoaderManagerService.EnvironmentType.UNIVERSAL; - if (modID == null) { - return dependencies; - } + for (int i = 0; i < mods.size(); i++) { + TomlTable modTable = mods.getTable(i); + if (modTable == null) continue; - TomlArray dependenciesArray = result.getArray("dependencies.\"" + modID + "\""); - if (dependenciesArray == null) { - return dependencies; - } + if (modId == null) modId = modTable.getString("modId"); - for (Object o : dependenciesArray.toList()) { - TomlTable mod = (TomlTable) o; - if (mod == null) continue; - String depId = mod.getString("modId"); - if (depId == null) continue; - dependencies.add(depId); - } + String v = modTable.getString("version"); + if (v != null && !v.equals("${file.jarVersion}")) version = v; - return dependencies; + TomlArray prov = modTable.getArray("provides"); + if (prov != null) { + for (int j = 0; j < prov.size(); j++) provides.add(prov.getString(j)); } - case "environment" -> { - LoaderManagerService.EnvironmentType environment = LoaderManagerService.EnvironmentType.UNIVERSAL; - - String modID = null; - for (Object o : modsArray.toList()) { - TomlTable mod = (TomlTable) o; - if (mod != null) { - modID = mod.getString("modId"); - } - } - - if (modID == null) { - return environment; - } + } - TomlArray dependenciesArray = result.getArray("dependencies.\"" + modID + "\""); - if (dependenciesArray == null) { - return environment; - } + if (modId != null) { + TomlArray depArray = result.getArray("deps.\"" + modId + "\""); + if (depArray != null) { + for (int i = 0; i < depArray.size(); i++) { + TomlTable depTable = depArray.getTable(i); + String depId = depTable.getString("modId"); + if (depId == null) continue; - for (Object o : dependenciesArray.toList()) { - TomlTable mod = (TomlTable) o; - if (mod == null) continue; - String depId = mod.getString("modId"); - if (depId == null) continue; // we only check for minecraft, neoforge and forge - if (!depId.equals("minecraft") && !depId.equals("neoforge") && !depId.equals("forge")) continue; - String depEnv = mod.getString("side"); - if (depEnv == null) continue; - switch (depEnv.toLowerCase()) { - case "client" -> environment = LoaderManagerService.EnvironmentType.CLIENT; - case "server" -> environment = LoaderManagerService.EnvironmentType.SERVER; - } + deps.add(depId); - if (environment != LoaderManagerService.EnvironmentType.UNIVERSAL) { - return environment; + // Determine Environment based on Minecraft/Forge side requirement + if (isPlatformId(depId)) { + String side = depTable.getString("side"); + if ("client".equalsIgnoreCase(side)) env = LoaderManagerService.EnvironmentType.CLIENT; + else if ("server".equalsIgnoreCase(side)) env = LoaderManagerService.EnvironmentType.SERVER; } } - - return environment; } } + return new ModMetadata(modId, version, provides, deps, env); } catch (Exception e) { - LOGGER.error("Error parsing TOML metadata: {}", e.getMessage()); + LOGGER.error("TOML Parse Error: {}", e.getMessage()); + return null; } - - return infoType.equals("version") || infoType.equals("modId") || infoType.equals("environment") ? null : Set.of(); } - private static Object getModInfoFromJson(BufferedReader reader, String infoType) { - JsonObject json = GSON.fromJson(reader, JsonObject.class); + private static ModMetadata parseJsonMetadata(BufferedReader reader) { + try { + JsonObject json = GSON.fromJson(reader, JsonObject.class); + JsonObject root = json; - switch (infoType) { - case "version" -> { - if (json.has("version")) { - return json.get("version").getAsString(); - } else if (json.has("quilt_loader") && json.get("quilt_loader").getAsJsonObject().has("version")) { - return json.get("quilt_loader").getAsJsonObject().get("version").getAsString(); - } + if (json.has("quilt_loader")) { + root = json.getAsJsonObject("quilt_loader"); } - case "modId" -> { - if (json.has("id")) { - return json.get("id").getAsString(); - } else if (json.has("quilt_loader") && json.get("quilt_loader").getAsJsonObject().has("id")) { - return json.get("quilt_loader").getAsJsonObject().get("id").getAsString(); - } - } - case "provides" -> { - Set providedIDs = new HashSet<>(); - if (json.has("provides")) { - for (JsonElement provides : json.get("provides").getAsJsonArray()) { - providedIDs.add(provides.getAsString()); - } - } else if (json.has("quilt_loader") && json.get("quilt_loader").getAsJsonObject().has("provides")) { - JsonObject quiltLoader = json.get("quilt_loader").getAsJsonObject(); - for (JsonElement provides : quiltLoader.get("provides").getAsJsonArray()) { - JsonObject providesObject = provides.getAsJsonObject(); - String id = providesObject.get("id").getAsString(); - providedIDs.add(id); - } + + String modId = getJsonString(root, "id"); + String version = getJsonString(root, "version"); + Set provides = new HashSet<>(); + Set deps = new HashSet<>(); + LoaderManagerService.EnvironmentType env = LoaderManagerService.EnvironmentType.UNIVERSAL; + + if (root.has("provides")) { + for (JsonElement e : root.get("provides").getAsJsonArray()) { + if (e.isJsonObject()) provides.add(e.getAsJsonObject().get("id").getAsString()); + else provides.add(e.getAsString()); } - return providedIDs; } - case "dependencies" -> { - Set dependencies = new HashSet<>(); - if (json.has("depends")) { - JsonObject depends = json.get("depends").getAsJsonObject(); - if (depends != null) { // Dont use asMap() since its only on gson 2.10^ - forge 1.18 - dependencies.addAll(depends.entrySet().stream().map(Map.Entry::getKey).toList()); - } - } else if (json.has("quilt_loader") && json.get("quilt_loader").getAsJsonObject().has("depends")) { - JsonObject depends = json.get("quilt_loader").getAsJsonObject().get("depends").getAsJsonObject(); - if (depends != null) { // Dont use asMap() since its only on gson 2.10^ - forge 1.18 - dependencies.addAll(depends.entrySet().stream().map(Map.Entry::getKey).toList()); + + if (root.has("depends")) { + JsonElement depends = root.get("depends"); + if (depends.isJsonObject()) { + deps.addAll(depends.getAsJsonObject().keySet()); + } else if (depends.isJsonArray()) { + for (JsonElement e : depends.getAsJsonArray()) { + if (e.isJsonObject()) deps.add(e.getAsJsonObject().get("id").getAsString()); + else deps.add(e.getAsString()); } } - return dependencies; } - case "environment" -> { - if (json.has("environment")) { - String environment = json.get("environment").getAsString(); - if (environment == null) return LoaderManagerService.EnvironmentType.UNIVERSAL; - return switch (environment.toLowerCase()) { - case "client" -> LoaderManagerService.EnvironmentType.CLIENT; - case "server" -> LoaderManagerService.EnvironmentType.SERVER; - default -> LoaderManagerService.EnvironmentType.UNIVERSAL; - }; - } else if (json.has("quilt_loader") && json.has("minecraft") && json.get("minecraft").getAsJsonObject().has("environment")) { - String environment = json.get("minecraft").getAsJsonObject().get("environment").getAsString(); - if (environment == null) return LoaderManagerService.EnvironmentType.UNIVERSAL; - return switch (environment.toLowerCase()) { - case "client" -> LoaderManagerService.EnvironmentType.CLIENT; - case "server" -> LoaderManagerService.EnvironmentType.SERVER; - default -> LoaderManagerService.EnvironmentType.UNIVERSAL; - }; - } + + if (root.has("environment")) { + String envStr = root.get("environment").getAsString(); + if ("client".equalsIgnoreCase(envStr)) env = LoaderManagerService.EnvironmentType.CLIENT; + else if ("server".equalsIgnoreCase(envStr)) env = LoaderManagerService.EnvironmentType.SERVER; } + + return new ModMetadata(modId, version, provides, deps, env); + } catch (Exception e) { + LOGGER.error("JSON Parse Error: {}", e.getMessage()); + return null; } + } + + private static boolean isPlatformId(String id) { + return "minecraft".equals(id) || "neoforge".equals(id) || "forge".equals(id); + } + + private static String getJsonString(JsonObject obj, String key) { + return obj.has(key) ? obj.get(key).getAsString() : null; + } + + public static Path getMetadataPath(FileSystem fs) { + String preferredEntry = switch (LOADER) { + case "neoforge" -> "META-INF/neoforge.mods.toml"; + case "fabric" -> "fabric.mod.json"; + case "forge" -> "META-INF/mods.toml"; + case "quilt" -> "quilt.mod.json"; + default -> null; + }; - return infoType.equals("version") || infoType.equals("modId") || infoType.equals("environment") ? null : Set.of(); + if (preferredEntry != null) { + Path p = fs.getPath(preferredEntry); + if (Files.exists(p)) return p; + } + + for (String fallback : List.of("META-INF/neoforge.mods.toml", "fabric.mod.json", "META-INF/mods.toml", "quilt.mod.json")) { + if (fallback.equals(preferredEntry)) continue; + Path p = fs.getPath(fallback); + if (Files.exists(p)) return p; + } + return null; + } + + private static boolean isMetadataFilename(String name) { + return name.endsWith("mods.toml") || name.endsWith("mod.json"); + } + + private static final Set KNOWN_SERVICES = Set.of( + "META-INF/services/net.minecraftforge.forgespi.locating.IModLocator", + "META-INF/services/net.minecraftforge.forgespi.locating.IDependencyLocator", + "META-INF/services/net.minecraftforge.forgespi.language.IModLanguageProvider", + "META-INF/services/net.neoforged.neoforgespi.locating.IModLocator", + "META-INF/services/net.neoforged.neoforgespi.locating.IDependencyLocator", + "META-INF/services/net.neoforged.neoforgespi.locating.IModLanguageLoader", + "META-INF/services/net.neoforged.neoforgespi.locating.IModFileCandidateLocator", + "META-INF/services/net.neoforged.neoforgespi.earlywindow.GraphicsBootstrapper" + ); + + public static boolean hasSpecificServices(FileSystem fs) { + // Fast Check + for (String service : KNOWN_SERVICES) { + if (Files.exists(fs.getPath(service))) return true; + } + + return false; } private static final String forbiddenChars = "\\/:*\"<>|!?&%$;=+"; diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java b/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java index 999e0b5a7..ccb3d18fa 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/WorkaroundUtil.java @@ -3,6 +3,9 @@ import pl.skidam.automodpack_core.Constants; import pl.skidam.automodpack_core.config.Jsons; +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; import java.nio.file.Path; import java.util.HashSet; import java.util.Set; @@ -17,7 +20,7 @@ public WorkaroundUtil(Path modapckPath) { // returns list of formatted modpack files which are mods with services (these mods need special treatment in order to work properly) // mods returned by this method should be installed in standard `~/mods/` directory - public Set getWorkaroundMods(Jsons.ModpackContentFields modpackContentFields) { + public Set getWorkaroundMods(Jsons.ModpackContentFields modpackContentFields) throws IOException { Set workaroundMods = new HashSet<>(); // this workaround is needed only for neo/forge mods @@ -28,12 +31,14 @@ public Set getWorkaroundMods(Jsons.ModpackContentFields modpackContentFi for (Jsons.ModpackContentFields.ModpackContentItem item : modpackContentFields.list) { if (item.type.equals("mod")) { Path modPath = SmartFileUtils.getPath(modpackPath, item.file); - if (FileInspection.hasSpecificServices(modPath)) { - workaroundMods.add(item.file); + try (FileSystem fs = FileSystems.newFileSystem(modPath)) { + if (FileInspection.hasSpecificServices(fs)) { + workaroundMods.add(item.file); + } } } } return workaroundMods; } -} +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/ModFileCache.java b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/ModFileCache.java new file mode 100644 index 000000000..2f6472d5c --- /dev/null +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/ModFileCache.java @@ -0,0 +1,122 @@ +package pl.skidam.automodpack_core.utils.cache; + +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; +import pl.skidam.automodpack_core.utils.FileInspection; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static pl.skidam.automodpack_core.Constants.LOGGER; + +public class ModFileCache implements AutoCloseable { + + private static final Map INSTANCES = new HashMap<>(); + private static final Object GLOBAL_LOCK = new Object(); + + private final Path dbPath; + private final MVStore store; + private final MVMap modFileMap; + private final AtomicInteger refCount = new AtomicInteger(1); + + private final Object[] locks = new Object[64]; + + public static ModFileCache open(Path path) { + Path absPath = path.toAbsolutePath().normalize(); + synchronized (GLOBAL_LOCK) { + ModFileCache existing = INSTANCES.get(absPath); + if (existing != null) { + existing.refCount.incrementAndGet(); + return existing; + } + + ModFileCache newCache = new ModFileCache(absPath); + INSTANCES.put(absPath, newCache); + return newCache; + } + } + + private ModFileCache(Path dbPath) { + this.dbPath = dbPath; + this.store = new MVStore.Builder() + .fileName(dbPath.toString()) + .cacheSize(20) + .open(); + + this.modFileMap = store.openMap("mod_file_data"); + + for (int i = 0; i < locks.length; i++) { + locks[i] = new Object(); + } + } + + public FileInspection.Mod getOrComputeMod(Path file, FileMetadataCache cache) throws IOException { + Path absPath = file.toAbsolutePath().normalize(); + String pathKey = absPath.toString(); + + String hash = cache.getOrComputeHash(absPath); + FileInspection.Mod cached = modFileMap.get(pathKey); + if (cached != null && hash.equalsIgnoreCase(cached.hash())) { + return cached; // CACHE HIT + } + + // Calculate which lock bucket to use + int lockIndex = Math.abs(pathKey.hashCode() % locks.length); + + synchronized (locks[lockIndex]) { + // Check if another thread has already updated the cache + hash = cache.getOrComputeHash(absPath); + cached = modFileMap.get(pathKey); + if (cached != null && hash.equalsIgnoreCase(cached.hash())) { + return cached; // CACHE HIT + } + + // Actual work happens here + FileInspection.Mod modFile = FileInspection.getMod(absPath, cache); + + if (modFile != null) { + modFileMap.put(pathKey, modFile); + } + + return modFile; + } + } + + public FileInspection.Mod getModOrNull(Path path, FileMetadataCache cache) { + try { + return getOrComputeMod(path, cache); + } catch (IOException e) { + LOGGER.error("Failed to compute hash for path: {}", path, e); + return null; + } + } + + // TODO: Consider running periodically + public void cleanup() { + synchronized (store) { + modFileMap.keySet().removeIf(pathString -> Files.notExists(Path.of(pathString))); + store.commit(); + store.compactFile(2000); + } + } + + @Override + public void close() { + synchronized (GLOBAL_LOCK) { + if (refCount.decrementAndGet() <= 0) { + try { + if (!store.isClosed()) { + store.commit(); + store.close(); + } + } finally { + INSTANCES.remove(this.dbPath, this); + } + } + } + } +} \ No newline at end of file diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index a609355f9..0a3b72c7f 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -7,6 +7,7 @@ import pl.skidam.automodpack_core.protocol.DownloadClient; import pl.skidam.automodpack_core.utils.*; import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; +import pl.skidam.automodpack_core.utils.cache.ModFileCache; import pl.skidam.automodpack_core.utils.launchers.LauncherVersionSwapper; import pl.skidam.automodpack_loader_core.ReLauncher; import pl.skidam.automodpack_loader_core.screen.ScreenManager; @@ -445,43 +446,48 @@ private boolean applyModpack(FileMetadataCache cache) throws Exception { Set modpackMods = new HashSet<>(); Collection modpackModList = new ArrayList<>(); - Path modpackModsDir = modpackDir.resolve("mods"); - if (Files.exists(modpackModsDir)) { - try (Stream stream = Files.list(modpackModsDir)) { - stream.forEach(path -> { - modpackMods.add(path); - FileInspection.Mod mod = FileInspection.getMod(path, cache); - if (mod != null) { - modpackModList.add(mod); - } - }); + Collection standardModList = new ArrayList<>(); + boolean needsRestart2; + Set ignoredFiles; + + try (var modCache = ModFileCache.open(modCacheDBFile)) { + Path modpackModsDir = modpackDir.resolve("mods"); + if (Files.exists(modpackModsDir)) { + try (Stream stream = Files.list(modpackModsDir)) { + stream.forEach(path -> { + modpackMods.add(path); + FileInspection.Mod mod = modCache.getModOrNull(path, cache); + if (mod != null) { + modpackModList.add(mod); + } + }); + } } - } - Collection standardModList = new ArrayList<>(); - Path standardModsDir = MODS_DIR; - if (Files.exists(standardModsDir)) { - try (Stream stream = Files.list(standardModsDir)) { - stream.forEach(path -> { - FileInspection.Mod mod = FileInspection.getMod(path, cache); - if (mod != null) { - standardModList.add(mod); - } - }); + Path standardModsDir = MODS_DIR; + if (Files.exists(standardModsDir)) { + try (Stream stream = Files.list(standardModsDir)) { + stream.forEach(path -> { + FileInspection.Mod mod = modCache.getModOrNull(path, cache); + if (mod != null) { + standardModList.add(mod); + } + }); + } } - } - // Check if the conflicting mods still exits, they might have been deleted by methods above - conflictingNestedMods = conflictingNestedMods.stream() - .filter(conflictingMod -> modpackMods.contains(conflictingMod.modPath())) - .toList(); + // Check if the conflicting mods still exits, they might have been deleted by methods above + conflictingNestedMods = conflictingNestedMods.stream() + .filter(conflictingMod -> modpackMods.contains(conflictingMod.path())) + .toList(); - if (!conflictingNestedMods.isEmpty()) { - LOGGER.warn("Found conflicting nested mods: {}", conflictingNestedMods); - } + if (!conflictingNestedMods.isEmpty()) { + LOGGER.warn("Found conflicting nested mods: {}", conflictingNestedMods); + } - boolean needsRestart2 = ModpackUtils.fixNestedMods(conflictingNestedMods, standardModList, cache); - Set ignoredFiles = ModpackUtils.getIgnoredFiles(conflictingNestedMods, workaroundMods); + needsRestart2 = ModpackUtils.fixNestedMods(conflictingNestedMods, standardModList, cache, modCache); + ignoredFiles = ModpackUtils.getIgnoredFiles(conflictingNestedMods, workaroundMods); + } Set forceCopyFiles = modpackContent.list.stream() .filter(item -> item.forceCopy) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java index 4389846ac..e202a7e86 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java @@ -10,6 +10,7 @@ import pl.skidam.automodpack_core.utils.FileInspection; import pl.skidam.automodpack_core.utils.ModpackContentTools; import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; +import pl.skidam.automodpack_core.utils.cache.ModFileCache; import pl.skidam.automodpack_loader_core.screen.ScreenManager; import java.io.*; @@ -298,25 +299,25 @@ public static boolean removeRestModsNotToCopy(Jsons.ModpackContentFields serverM // Copies necessary nested mods from modpack mods to standard mods folder // Returns true if requires client restart - public static boolean fixNestedMods(List conflictingNestedMods, Collection standardModList, FileMetadataCache cache) throws IOException { + public static boolean fixNestedMods(List conflictingNestedMods, Collection standardModList, FileMetadataCache cache, ModFileCache modCache) throws IOException { if (conflictingNestedMods.isEmpty()) return false; - final List standardModIDs = standardModList.stream().map(FileInspection.Mod::modID).toList(); + final List standardModIDs = standardModList.stream().flatMap(mod -> mod.IDs().stream()).toList(); boolean needsRestart = false; for (FileInspection.Mod mod : conflictingNestedMods) { // Check mods provides, if there's some mod which is named with the same id as some other mod 'provides' remove the mod which provides that id as well, otherwise loader will crash - if (standardModIDs.stream().anyMatch(mod.providesIDs()::contains)) + if (standardModIDs.stream().anyMatch(mod.IDs()::contains)) continue; - Path modPath = mod.modPath(); + Path modPath = mod.path(); Path standardModPath = MODS_DIR.resolve(modPath.getFileName()); if (!Files.exists(standardModPath) || !mod.hash().equalsIgnoreCase(cache.getHashOrNull(standardModPath))) { needsRestart = true; LOGGER.info("Copying nested mod {} to standard mods folder", standardModPath.getFileName()); SmartFileUtils.copyFile(modPath, standardModPath); - var newMod = FileInspection.getMod(standardModPath, cache); + var newMod = modCache.getModOrNull(standardModPath, cache); if (newMod != null) standardModList.add(newMod); // important } } @@ -329,7 +330,7 @@ public static Set getIgnoredFiles(List conflictingNe Set newIgnoredFiles = new HashSet<>(workarounds); for (FileInspection.Mod mod : conflictingNestedMods) { - newIgnoredFiles.add(SmartFileUtils.formatPath(mod.modPath(), modpacksDir)); + newIgnoredFiles.add(SmartFileUtils.formatPath(mod.path(), modpacksDir)); } return newIgnoredFiles; @@ -341,9 +342,9 @@ public static Map getDupeMods(Path modpa final Map duplicates = new HashMap<>(); for (FileInspection.Mod modpackMod : modpackModList) { - FileInspection.Mod standardMod = standardModList.stream().filter(mod -> mod.modID().equals(modpackMod.modID())).findFirst().orElse(null); // There might be super rare edge case if client would have for some reason more than one mod with the same mod id + FileInspection.Mod standardMod = standardModList.stream().filter(mod -> mod.IDs().stream().anyMatch(modpackMod.IDs()::contains)).findFirst().orElse(null); if (standardMod != null) { - String formattedFile = SmartFileUtils.formatPath(modpackMod.modPath(), modpackDir); + String formattedFile = SmartFileUtils.formatPath(modpackMod.path(), modpackDir); if (ignoredMods.contains(formattedFile) || forceCopyFiles.contains(formattedFile)) continue; @@ -378,10 +379,7 @@ public static RemoveDupeModsResult removeDupeMods(Path modpackDir, Collection idsToKeep = new HashSet<>(); - modsToKeep.forEach(mod -> { - idsToKeep.add(mod.modID()); - idsToKeep.addAll(mod.providesIDs()); - }); + modsToKeep.forEach(mod -> idsToKeep.addAll(mod.IDs())); boolean requiresRestart = false; Set dependentMods = new HashSet<>(); @@ -390,13 +388,11 @@ public static RemoveDupeModsResult removeDupeMods(Path modpackDir, Collection providesIDs = modpackMod.providesIDs(); - List IDs = new ArrayList<>(providesIDs); - IDs.add(modId); + List IDs = new ArrayList<>(modpackMod.IDs()); + String modId = IDs.get(0); boolean isDependent = IDs.stream().anyMatch(idsToKeep::contains); boolean isWorkaround = workaroundMods.contains(formatedPath); @@ -409,7 +405,7 @@ public static RemoveDupeModsResult removeDupeMods(Path modpackDir, Collection modList, Set modsToKeep) { - for (String depId : mod.dependencies()) { + for (String depId : mod.deps()) { for (FileInspection.Mod modItem : modList) { - if ((modItem.modID().equals(depId) || modItem.providesIDs().contains(depId)) && modsToKeep.add(modItem)) { + if (modItem.IDs().stream().anyMatch(s -> s.equalsIgnoreCase(depId)) && modsToKeep.add(modItem)) { addDependenciesRecursively(modItem, modList, modsToKeep); } } diff --git a/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java b/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java index 84cf28f7b..f0e4650d3 100644 --- a/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java +++ b/loader/fabric/15/src/main/java/pl/skidam/automodpack_loader_core_fabric_15/mods/ModpackLoader15.java @@ -11,7 +11,6 @@ import net.fabricmc.loader.impl.metadata.DependencyOverrides; import net.fabricmc.loader.impl.metadata.VersionOverrides; import net.fabricmc.loader.impl.util.SystemProperties; -import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; @@ -152,14 +151,23 @@ public List getModpackNestedConflicts(Path modpackDir, FileM if (hash == null) continue; + Set modIds = new HashSet<>(); + modIds.add(mod.getId()); + modIds.addAll(mod.getProvides()); + + Set deps = new HashSet<>(); + for (ModDependency dep : mod.getDependencies()) { + deps.add(dep.getModId()); + } + FileInspection.Mod conflictingMod = new FileInspection.Mod( - mod.getId(), + modIds, hash, - mod.getProvides(), mod.getVersion().getFriendlyString(), path, - LoaderManagerService.EnvironmentType.UNIVERSAL, - mod.getDependencies().stream().map(ModDependency::getModId).toList()); + deps, + Set.of() + ); conflictingNestedMods.add(conflictingMod); } diff --git a/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java b/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java index c4ff76621..bb29a66db 100644 --- a/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java +++ b/loader/fabric/16/src/main/java/pl/skidam/automodpack_loader_core_fabric_16/mods/ModpackLoader16.java @@ -11,7 +11,6 @@ import net.fabricmc.loader.impl.metadata.DependencyOverrides; import net.fabricmc.loader.impl.metadata.VersionOverrides; import net.fabricmc.loader.impl.util.SystemProperties; -import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.loader.ModpackLoaderService; import pl.skidam.automodpack_core.utils.FileInspection; import pl.skidam.automodpack_core.utils.cache.FileMetadataCache; @@ -152,14 +151,23 @@ public List getModpackNestedConflicts(Path modpackDir, FileM if (hash == null) continue; + Set modIds = new HashSet<>(); + modIds.add(mod.getId()); + modIds.addAll(mod.getProvides()); + + Set deps = new HashSet<>(); + for (ModDependency dep : mod.getDependencies()) { + deps.add(dep.getModId()); + } + FileInspection.Mod conflictingMod = new FileInspection.Mod( - mod.getId(), + modIds, hash, - mod.getProvides(), mod.getVersion().getFriendlyString(), path, - LoaderManagerService.EnvironmentType.UNIVERSAL, - mod.getDependencies().stream().map(ModDependency::getModId).toList()); + deps, + Set.of() + ); conflictingNestedMods.add(conflictingMod); } From a63537b9902e653e1d1afe942591665c180a8157 Mon Sep 17 00:00:00 2001 From: skidam Date: Sun, 25 Jan 2026 16:43:24 +0100 Subject: [PATCH 10/22] stuf --- .../client/ModpackUpdater.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index 0a3b72c7f..6c2f1c748 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -167,7 +167,6 @@ private void CheckAndLoadModpack(FileMetadataCache cache) throws Exception { LOGGER.info("Modpack is already loaded"); } - // TODO split it into different methods, its too long public void startUpdate(Set filesToUpdate) { if (modpackSecret == null) { LOGGER.error("Cannot update modpack, secret is null"); @@ -189,7 +188,7 @@ public void startUpdate(Set files // FETCH long startFetching = System.currentTimeMillis(); - List fetchDatas = new LinkedList<>(); + List fetchDatas = new ArrayList<>(); for (Jsons.ModpackContentFields.ModpackContentItem serverItem : finalFilesToUpdate) { @@ -202,11 +201,12 @@ public void startUpdate(Set files } } - fetchManager = new FetchManager(fetchDatas); - new ScreenManager().fetch(fetchManager); - fetchManager.fetch(); - LOGGER.info("Finished fetching urls in {}ms", System.currentTimeMillis() - startFetching); - + if (!fetchDatas.isEmpty()) { + fetchManager = new FetchManager(fetchDatas); + new ScreenManager().fetch(fetchManager); + fetchManager.fetch(); + LOGGER.info("Finished fetching urls in {}ms", System.currentTimeMillis() - startFetching); + } // DOWNLOAD try { From 1743bfc4c5c874f6cd3e8326873536a9466d2722 Mon Sep 17 00:00:00 2001 From: skidam Date: Mon, 26 Jan 2026 14:24:11 +0100 Subject: [PATCH 11/22] lesser evil --- .../client/ModpackUpdater.java | 243 +++++++++--------- 1 file changed, 125 insertions(+), 118 deletions(-) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index 6c2f1c748..980b7c36f 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -228,7 +228,7 @@ public void startUpdate(Set files for (var download : failedDownloads.entrySet()) { var item = download.getKey(); var urls = download.getValue(); - LOGGER.error("{}{}", "Failed to download: " + item.file + " from ", urls); + LOGGER.error("Failed to download: {} from {}", item.file, urls); failedFiles.append(item.file); } @@ -256,11 +256,123 @@ public void startUpdate(Set files private void downloadModpack(Set finalFilesToUpdate, long startFetching, FileMetadataCache cache) throws InterruptedException { int wholeQueue = finalFilesToUpdate.size(); - if (wholeQueue > 0) { - LOGGER.info("In queue left {} files to download ({}MB)", wholeQueue, totalBytesToDownload / 1024 / 1024); + if (wholeQueue == 0) { + LOGGER.info("No files to download."); + return; + } + + LOGGER.info("In queue left {} files to download ({}MB)", wholeQueue, totalBytesToDownload / 1024 / 1024); + + DownloadClient downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), + Math.min(wholeQueue, 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); + if (downloadClient == null) { + return; + } + + downloadManager = new DownloadManager(totalBytesToDownload); + new ScreenManager().download(downloadManager, getModpackName()); + downloadManager.attachDownloadClient(downloadClient); + + var randomizedList = new ArrayList<>(finalFilesToUpdate); + Collections.shuffle(randomizedList); + for (var serverItem : randomizedList) { + + String serverFilePath = serverItem.file; + String serverHash = serverItem.sha1; + + Path downloadFile = SmartFileUtils.getPath(modpackDir, serverFilePath); + + if (!Files.exists(downloadFile)) { + newDownloadedFiles.add(serverFilePath); + } + + List urls = new ArrayList<>(); + if (fetchManager.getFetchDatas().containsKey(serverHash)) { + urls.addAll(fetchManager.getFetchDatas().get(serverHash).fetchedData().urls()); + } + + Runnable failureCallback = () -> { + failedDownloads.put(serverItem, urls); + }; + + Runnable successCallback = () -> { + List mainPageUrls = new LinkedList<>(); + if (fetchManager != null && fetchManager.getFetchDatas().get(serverHash) != null) { + mainPageUrls = fetchManager.getFetchDatas().get(serverHash).fetchedData().mainPageUrls(); + } - DownloadClient downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), - Math.min(wholeQueue, 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); + changelogs.changesAddedList.put(downloadFile.getFileName().toString(), mainPageUrls); + + try { + cache.overwriteCache(downloadFile, serverHash); + } catch (Exception e) { + LOGGER.error("Failed to update cache for {}", downloadFile, e); + } + }; + + + downloadManager.download(downloadFile, serverHash, urls, successCallback, failureCallback); + } + + downloadManager.joinAll(); + + LOGGER.info("Finished downloading files in {}ms", System.currentTimeMillis() - startFetching); + + if (downloadManager.isCanceled()) { + LOGGER.warn("Download canceled"); + return; + } + + downloadManager.cancelAllAndShutdown(); + totalBytesToDownload = 0; + + if (failedDownloads.isEmpty()) { + return; + } + + Map hashesToRefresh = new HashMap<>(); // File name, hash + var failedDownloadsSecMap = new HashMap<>(failedDownloads); + failedDownloadsSecMap.forEach((k, v) -> { + hashesToRefresh.put(k.file, k.sha1); + failedDownloads.remove(k); + totalBytesToDownload += Long.parseLong(k.size); + }); + + if (hashesToRefresh.isEmpty()) { + return; + } + + LOGGER.warn("Failed to download {} files", hashesToRefresh.size()); + + // make byte[][] from hashesToRefresh.values() + byte[][] hashesArray = hashesToRefresh.values().stream() + .map(String::getBytes) + .toArray(byte[][]::new); + + // send it to the server and get the new modpack content + // TODO set client to a waiting for the server to respond screen + LOGGER.warn("Trying to refresh the modpack content"); + LOGGER.info("Sending hashes to refresh: {}", hashesToRefresh.values()); + var refreshedContentOptional = ModpackUtils.refreshServerModpackContent(modpackAddresses, modpackSecret, hashesArray, false); + if (refreshedContentOptional.isEmpty()) { + LOGGER.error("Failed to refresh the modpack content"); + } else { + LOGGER.info("Successfully refreshed the modpack content"); + // retry the download + // success + // or fail and then show the error + + var refreshedContent = refreshedContentOptional.get(); + this.serverModpackContent = refreshedContent; + this.serverModpackContentJson = GSON.toJson(refreshedContent); + + // filter list to only the failed downloads + var refreshedFilteredList = refreshedContent.list.stream().filter(item -> hashesToRefresh.containsKey(item.file)).toList(); + if (refreshedFilteredList.isEmpty()) { + return; + } + + downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), Math.min(refreshedFilteredList.size(), 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); if (downloadClient == null) { return; } @@ -269,7 +381,9 @@ private void downloadModpack(Set new ScreenManager().download(downloadManager, getModpackName()); downloadManager.attachDownloadClient(downloadClient); - var randomizedList = new ArrayList<>(finalFilesToUpdate); + // TODO try to fetch again from modrinth and curseforge + + randomizedList = new ArrayList<>(refreshedFilteredList); Collections.shuffle(randomizedList); for (var serverItem : randomizedList) { @@ -278,26 +392,14 @@ private void downloadModpack(Set Path downloadFile = SmartFileUtils.getPath(modpackDir, serverFilePath); - if (!Files.exists(downloadFile)) { - newDownloadedFiles.add(serverFilePath); - } - - List urls = new ArrayList<>(); - if (fetchManager.getFetchDatas().containsKey(serverHash)) { - urls.addAll(fetchManager.getFetchDatas().get(serverHash).fetchedData().urls()); - } + LOGGER.info("Retrying to download {} from {}", serverFilePath, modpackAddresses.hostAddress.getHostName()); Runnable failureCallback = () -> { - failedDownloads.put(serverItem, urls); + failedDownloads.put(serverItem, List.of()); }; Runnable successCallback = () -> { - List mainPageUrls = new LinkedList<>(); - if (fetchManager != null && fetchManager.getFetchDatas().get(serverHash) != null) { - mainPageUrls = fetchManager.getFetchDatas().get(serverHash).fetchedData().mainPageUrls(); - } - - changelogs.changesAddedList.put(downloadFile.getFileName().toString(), mainPageUrls); + changelogs.changesAddedList.put(downloadFile.getFileName().toString(), null); try { cache.overwriteCache(downloadFile, serverHash); @@ -306,114 +408,19 @@ private void downloadModpack(Set } }; - - downloadManager.download(downloadFile, serverHash, urls, successCallback, failureCallback); + downloadManager.download(downloadFile, serverHash, List.of(), successCallback, failureCallback); } downloadManager.joinAll(); - LOGGER.info("Finished downloading files in {}ms", System.currentTimeMillis() - startFetching); - if (downloadManager.isCanceled()) { LOGGER.warn("Download canceled"); return; } downloadManager.cancelAllAndShutdown(); - totalBytesToDownload = 0; - - Map hashesToRefresh = new HashMap<>(); // File name, hash - var failedDownloadsSecMap = new HashMap<>(failedDownloads); - failedDownloadsSecMap.forEach((k, v) -> { - hashesToRefresh.put(k.file, k.sha1); - failedDownloads.remove(k); - totalBytesToDownload += Long.parseLong(k.size); - }); - - if (!hashesToRefresh.isEmpty()) { - LOGGER.warn("Failed to download {} files", hashesToRefresh.size()); - } - - if (!hashesToRefresh.isEmpty()) { - // make byte[][] from hashesToRefresh.values() - byte[][] hashesArray = hashesToRefresh.values().stream() - .map(String::getBytes) - .toArray(byte[][]::new); - - // send it to the server and get the new modpack content - // TODO set client to a waiting for the server to respond screen - LOGGER.warn("Trying to refresh the modpack content"); - LOGGER.info("Sending hashes to refresh: {}", hashesToRefresh.values()); - var refreshedContentOptional = ModpackUtils.refreshServerModpackContent(modpackAddresses, modpackSecret, hashesArray, false); - if (refreshedContentOptional.isEmpty()) { - LOGGER.error("Failed to refresh the modpack content"); - } else { - LOGGER.info("Successfully refreshed the modpack content"); - // retry the download - // success - // or fail and then show the error - - var refreshedContent = refreshedContentOptional.get(); - this.serverModpackContent = refreshedContent; - this.serverModpackContentJson = GSON.toJson(refreshedContent); - - // filter list to only the failed downloads - var refreshedFilteredList = refreshedContent.list.stream().filter(item -> hashesToRefresh.containsKey(item.file)).toList(); - if (refreshedFilteredList.isEmpty()) { - return; - } - - downloadClient = DownloadClient.tryCreate(modpackAddresses, modpackSecret.secretBytes(), Math.min(refreshedFilteredList.size(), 5), ModpackUtils.userValidationCallback(modpackAddresses.hostAddress, false)); - if (downloadClient == null) { - return; - } - downloadManager = new DownloadManager(totalBytesToDownload); - new ScreenManager().download(downloadManager, getModpackName()); - downloadManager.attachDownloadClient(downloadClient); - - // TODO try to fetch again from modrinth and curseforge - - randomizedList = new ArrayList<>(refreshedFilteredList); - Collections.shuffle(randomizedList); - for (var serverItem : randomizedList) { - - String serverFilePath = serverItem.file; - String serverHash = serverItem.sha1; - - Path downloadFile = SmartFileUtils.getPath(modpackDir, serverFilePath); - - LOGGER.info("Retrying to download {} from {}", serverFilePath, modpackAddresses.hostAddress.getHostName()); - - Runnable failureCallback = () -> { - failedDownloads.put(serverItem, List.of()); - }; - - Runnable successCallback = () -> { - changelogs.changesAddedList.put(downloadFile.getFileName().toString(), null); - - try { - cache.overwriteCache(downloadFile, serverHash); - } catch (Exception e) { - LOGGER.error("Failed to update cache for {}", downloadFile, e); - } - }; - - downloadManager.download(downloadFile, serverHash, List.of(), successCallback, failureCallback); - } - - downloadManager.joinAll(); - - if (downloadManager.isCanceled()) { - LOGGER.warn("Download canceled"); - return; - } - - downloadManager.cancelAllAndShutdown(); - - LOGGER.info("Finished refreshed downloading files in {}ms", System.currentTimeMillis() - startFetching); - } - } + LOGGER.info("Finished refreshed downloading files in {}ms", System.currentTimeMillis() - startFetching); } } From c0c9a54c4389fde7b265e55a3f223f56d605a3ba Mon Sep 17 00:00:00 2001 From: skidam Date: Mon, 26 Jan 2026 14:39:23 +0100 Subject: [PATCH 12/22] Fix nested services regression --- .../utils/FileInspection.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java index b3806d29e..ec8da3fc7 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java @@ -358,11 +358,51 @@ private static boolean isMetadataFilename(String name) { ); public static boolean hasSpecificServices(FileSystem fs) { - // Fast Check + // Fast Check: Look in the root FileSystem for (String service : KNOWN_SERVICES) { - if (Files.exists(fs.getPath(service))) return true; + if (Files.exists(fs.getPath(service))) { + return true; + } } + // Slow Check: Scan nested JARs in META-INF/jarjar + return hasSpecificServicesNested(fs); + } + + private static boolean hasSpecificServicesNested(FileSystem fs) { + Path jarJarDir = fs.getPath("META-INF", "jarjar"); + + if (Files.notExists(jarJarDir)) { + return false; + } + + try (DirectoryStream stream = Files.newDirectoryStream(jarJarDir, "*.jar")) { + for (Path nestedJar : stream) { + if (scanNestedJar(nestedJar)) { + return true; + } + } + } catch (IOException e) { + LOGGER.error("Error examining JarJar directory in {}", fs, e); + } + + return false; + } + + private static boolean scanNestedJar(Path nestedJarPath) { + try (InputStream is = Files.newInputStream(nestedJarPath); + BufferedInputStream bis = new BufferedInputStream(is); + ZipInputStream zip = new ZipInputStream(bis)) { + + ZipEntry entry; + while ((entry = zip.getNextEntry()) != null) { + if (KNOWN_SERVICES.contains(entry.getName())) { + return true; + } + } + } catch (IOException e) { + LOGGER.error("Error reading nested JAR {}: {}", nestedJarPath, e.getMessage()); + } return false; } From 497ddab9fc7d42eb84f9228da46888ed6e4dae39 Mon Sep 17 00:00:00 2001 From: skidam Date: Wed, 28 Jan 2026 16:39:42 +0100 Subject: [PATCH 13/22] temporary ugly path mapping fix --- .../skidam/automodpack_core/modpack/ModpackContent.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java index 2a4e2b678..454da242b 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java @@ -89,11 +89,17 @@ public boolean create(FileMetadataCache cache) { } } + var tempPathMap = new HashMap<>(filesToProcess); + List> futures = filesToProcess.entrySet().stream() .map(entry -> CompletableFuture.supplyAsync(() -> { try { var contentEntry = generateContent(entry.getValue(), entry.getKey(), cache); + if (contentEntry == null) { + return null; + } LOGGER.debug("Generated modpack content for {}", entry.getValue()); + tempPathMap.put(contentEntry.sha1, entry.getValue()); return contentEntry; } catch (Exception e) { LOGGER.error("Error generating content for {}", entry.getValue(), e); @@ -108,7 +114,7 @@ public boolean create(FileMetadataCache cache) { Jsons.ModpackContentFields.ModpackContentItem item = future.join(); if (item != null) { list.add(item); - pathsMap.put(item.sha1, SmartFileUtils.getPathFromCWD(item.file)); + pathsMap.put(item.sha1, tempPathMap.get(item.sha1)); } } From bad575bfd79a59ecb769d1aee1c19098c2bbe175 Mon Sep 17 00:00:00 2001 From: skidam Date: Wed, 28 Jan 2026 19:07:36 +0100 Subject: [PATCH 14/22] Annotate FetchManager as nullable and handle it --- .../automodpack_loader_core/client/ModpackUpdater.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index 980b7c36f..fb5ed6235 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -1,5 +1,6 @@ package pl.skidam.automodpack_loader_core.client; +import org.jetbrains.annotations.Nullable; import pl.skidam.automodpack_core.auth.Secrets; import pl.skidam.automodpack_core.auth.SecretsStore; import pl.skidam.automodpack_core.config.Jsons; @@ -28,7 +29,6 @@ public class ModpackUpdater { public Changelogs changelogs = new Changelogs(); public DownloadManager downloadManager; - public FetchManager fetchManager; public long totalBytesToDownload = 0; public boolean fullDownload = false; private Jsons.ModpackContentFields serverModpackContent; @@ -201,6 +201,8 @@ public void startUpdate(Set files } } + FetchManager fetchManager = null; + if (!fetchDatas.isEmpty()) { fetchManager = new FetchManager(fetchDatas); new ScreenManager().fetch(fetchManager); @@ -210,7 +212,7 @@ public void startUpdate(Set files // DOWNLOAD try { - downloadModpack(finalFilesToUpdate, startFetching, cache); + downloadModpack(finalFilesToUpdate, startFetching, fetchManager, cache); LOGGER.info("Done, saving {}", modpackContentFile); @@ -253,7 +255,7 @@ public void startUpdate(Set files } } - private void downloadModpack(Set finalFilesToUpdate, long startFetching, FileMetadataCache cache) throws InterruptedException { + private void downloadModpack(Set finalFilesToUpdate, long startFetching, @Nullable FetchManager fetchManager, FileMetadataCache cache) throws InterruptedException { int wholeQueue = finalFilesToUpdate.size(); if (wholeQueue == 0) { @@ -287,7 +289,7 @@ private void downloadModpack(Set } List urls = new ArrayList<>(); - if (fetchManager.getFetchDatas().containsKey(serverHash)) { + if (fetchManager != null && fetchManager.getFetchDatas().containsKey(serverHash)) { urls.addAll(fetchManager.getFetchDatas().get(serverHash).fetchedData().urls()); } From b46f79fd358450b49b5446c7c7bfedb955cad8a8 Mon Sep 17 00:00:00 2001 From: skidam Date: Fri, 30 Jan 2026 15:11:51 +0100 Subject: [PATCH 15/22] rethrow e and handle null downloadmanager in case --- .../skidam/automodpack_loader_core/client/ModpackUpdater.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index fb5ed6235..b0e928838 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -219,8 +219,8 @@ public void startUpdate(Set files // Downloads completed, save json files Files.writeString(modpackContentFile, serverModpackContentJson); } catch (Exception e) { - downloadManager.cancelAllAndShutdown(); - LOGGER.error("Error during modpack download", e); + if (downloadManager != null) downloadManager.cancelAllAndShutdown(); + throw e; } LegacyClientCacheUtils.deleteDummyFiles(); From 0b653e549e9c939f9bf418fef5a3386399d08a54 Mon Sep 17 00:00:00 2001 From: skidam Date: Fri, 30 Jan 2026 19:00:58 +0100 Subject: [PATCH 16/22] Implement nix-store like file management --- .../pl/skidam/automodpack_core/Constants.java | 1 + .../modpack/ModpackContent.java | 4 +- .../automodpack_core/protocol/NetUtils.java | 4 +- .../utils/FileInspection.java | 2 +- .../automodpack_core/utils/HashUtils.java | 108 ++++++ .../utils/LockFreeInputStream.java | 71 ++-- .../utils/SmartFileUtils.java | 290 +++++---------- .../utils/cache/FileMetadataCache.java | 4 +- .../utils/SmartFileUtilsTest.java | 10 +- .../automodpack_loader_core/SelfUpdater.java | 3 +- .../client/ModpackUpdater.java | 8 +- .../client/ModpackUtils.java | 67 ++-- .../utils/DownloadManager.java | 346 ++++++++++-------- 13 files changed, 488 insertions(+), 430 deletions(-) create mode 100644 core/src/main/java/pl/skidam/automodpack_core/utils/HashUtils.java diff --git a/core/src/main/java/pl/skidam/automodpack_core/Constants.java b/core/src/main/java/pl/skidam/automodpack_core/Constants.java index 1bd9ee0e1..d9bb65caa 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/Constants.java +++ b/core/src/main/java/pl/skidam/automodpack_core/Constants.java @@ -32,6 +32,7 @@ public class Constants { public static Jsons.ClientConfigFieldsV2 clientConfig; public static Jsons.KnownHostsFields knownHosts; public static final Path automodpackDir = Path.of("automodpack"); + public static final Path storeDir = automodpackDir.resolve("store"); public static final Path hostModpackDir = automodpackDir.resolve("host-modpack"); // TODO More server modpacks // Main - required diff --git a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java index 454da242b..bb9d67d86 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java +++ b/core/src/main/java/pl/skidam/automodpack_core/modpack/ModpackContent.java @@ -306,14 +306,14 @@ private Jsons.ModpackContentFields.ModpackContentItem generateContent(final Path type = "other"; } - String sha1 = cache != null ? cache.getHashOrNull(file) : SmartFileUtils.getHash(file); + String sha1 = cache != null ? cache.getHashOrNull(file) : HashUtils.getHash(file); // For CF API String murmur = null; if (type.equals("mod") || type.equals("shader") || type.equals("resourcepack")) { murmur = sha1MurmurMapPreviousContent.get(sha1); // Get from cache if (murmur == null) { - murmur = SmartFileUtils.getCurseforgeMurmurHash(file); + murmur = HashUtils.getCurseforgeMurmurHash(file); } } diff --git a/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java b/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java index cf5cd80ab..f3b60e698 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/protocol/NetUtils.java @@ -100,7 +100,7 @@ public static void saveCertificate(X509Certificate cert, Path path) throws Excep String certPem = "-----BEGIN CERTIFICATE-----\n" + formatBase64(Base64.getEncoder().encodeToString(cert.getEncoded())) + "-----END CERTIFICATE-----"; - SmartFileUtils.setupFilePaths(path); + SmartFileUtils.createParentDirs(path); Files.writeString(path, certPem); } @@ -117,7 +117,7 @@ public static void savePrivateKey(PrivateKey key, Path path) throws Exception { String keyPem = "-----BEGIN PRIVATE KEY-----\n" + formatBase64(Base64.getEncoder().encodeToString(keySpec.getEncoded())) + "-----END PRIVATE KEY-----"; - SmartFileUtils.setupFilePaths(path); + SmartFileUtils.createParentDirs(path); Files.writeString(path, keyPem); } diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java index ec8da3fc7..52de147e5 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/FileInspection.java @@ -54,7 +54,7 @@ private record ModMetadata(String modId, String version, Set provides, S public static Mod getMod(Path file, FileMetadataCache cache) { if (isJarInvalid(file)) return null; - String hash = cache != null ? cache.getHashOrNull(file) : SmartFileUtils.getHash(file); + String hash = cache != null ? cache.getHashOrNull(file) : HashUtils.getHash(file); if (hash == null) { LOGGER.error("Failed to get hash for file: {}", file); return null; diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/HashUtils.java b/core/src/main/java/pl/skidam/automodpack_core/utils/HashUtils.java new file mode 100644 index 000000000..87cca26aa --- /dev/null +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/HashUtils.java @@ -0,0 +1,108 @@ +package pl.skidam.automodpack_core.utils; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.HexFormat; + +import static pl.skidam.automodpack_core.Constants.LOGGER; + +public class HashUtils { + + public static String getHash(Path path) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + try (InputStream is = new LockFreeInputStream(path)) { + byte[] buffer = new byte[64 * 1024]; + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + digest.update(buffer, 0, bytesRead); + } + } + return HexFormat.of().formatHex(digest.digest()); + } catch (IOException ignored) { + // File might not exist + } catch (Exception e) { + LOGGER.error("Failed to get hash for path: {}", path, e); + } + return null; + } + + /** + * Calculates the CurseForge specific MurmurHash2. + * Normalized by ignoring whitespace (0x9, 0xA, 0xD, 0x20). + */ + public static String getCurseforgeMurmurHash(Path file) throws IOException { + if (!Files.exists(file)) return null; + + // MurmurHash2 Constants + final int m = 0x5bd1e995; + final int r = 24; + final int seed = 1; + + // Pass 1: Count valid non-whitespace bytes to determine the hash seed + long validLength = 0; + byte[] buffer = new byte[64 * 1024]; + + try (InputStream is = new LockFreeInputStream(file)) { + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + for (int i = 0; i < bytesRead; i++) { + if (!isWhitespace(buffer[i])) { + validLength++; + } + } + } + } + + // Pass 2: Calculate Hash + long h = (seed ^ validLength); + long k = 0; + int shift = 0; + + try (InputStream is = new LockFreeInputStream(file)) { + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + for (int i = 0; i < bytesRead; i++) { + byte b = buffer[i]; + if (isWhitespace(b)) continue; + + // Append byte to current 4-byte chunk + k = k | ((long) (b & 0xFF) << shift); + shift += 8; + + if (shift == 32) { + k = (k * m) & 0xFFFFFFFFL; + k ^= (k >>> r); + k = (k * m) & 0xFFFFFFFFL; + + h = (h * m) & 0xFFFFFFFFL; + h ^= k; + + // Reset chunk + k = 0; + shift = 0; + } + } + } + } + + // Handle tail + if (shift > 0) { + h ^= k; + h = (h * m) & 0xFFFFFFFFL; + } + + h ^= (h >>> 13); + h = (h * m) & 0xFFFFFFFFL; + h ^= (h >>> 15); + + return String.valueOf(h); + } + + private static boolean isWhitespace(byte b) { + return b == 0x9 || b == 0xA || b == 0xD || b == 0x20; + } +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java b/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java index 870865711..8033994e8 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/LockFreeInputStream.java @@ -4,25 +4,18 @@ import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; +import java.nio.channels.FileChannel; +import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import static pl.skidam.automodpack_core.utils.SmartFileUtils.isFilePhysical; - -/** - * A safe input stream that can read files currently locked or in-use by other processes (like active game logs). - *

- * It automatically applies a workaround for Windows file locking issues ("Access Denied") while using - * standard, high-performance NIO on all other platforms. - */ public class LockFreeInputStream extends InputStream { private final InputStream delegate; public LockFreeInputStream(Path path) throws IOException { - if (PlatformUtils.IS_WIN && isFilePhysical(path)) { + if (useRandomAccess(path)) { RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r"); this.delegate = Channels.newInputStream(raf.getChannel()); } else { @@ -30,46 +23,34 @@ public LockFreeInputStream(Path path) throws IOException { } } - public static ReadableByteChannel openChannel(Path path) throws IOException { - if (PlatformUtils.IS_WIN && isFilePhysical(path)) { + /** + * Opens a FileChannel optimized for locked files. + *

+ * Returns a concrete FileChannel. + * This supports both Netty's ChunkedNioStream and zero-copy file transfers. + */ + public static FileChannel openChannel(Path path) throws IOException { + if (useRandomAccess(path)) { + // RandomAccessFile.getChannel() returns a FileChannel return new RandomAccessFile(path.toFile(), "r").getChannel(); } else { - return Files.newByteChannel(path, StandardOpenOption.READ); + // FileChannel.open() returns a FileChannel + return FileChannel.open(path, StandardOpenOption.READ); } } - @Override - public int read() throws IOException { - return delegate.read(); - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - return delegate.read(b, off, len); - } - - @Override - public int available() throws IOException { - return delegate.available(); - } - - @Override - public void close() throws IOException { - delegate.close(); - } - - @Override - public synchronized void mark(int readlimit) { - delegate.mark(readlimit); + // Windows locks files aggressively. RandomAccessFile is often more lenient than NIO Files.newByteChannel. + // We also ensure the file is on the default filesystem (not inside a Zip, etc). + private static boolean useRandomAccess(Path path) { + return PlatformUtils.IS_WIN && path.getFileSystem() == FileSystems.getDefault(); } - @Override - public synchronized void reset() throws IOException { - delegate.reset(); - } - - @Override - public boolean markSupported() { - return delegate.markSupported(); - } + // ... Standard InputStream overrides below ... + @Override public int read() throws IOException { return delegate.read(); } + @Override public int read(byte[] b, int off, int len) throws IOException { return delegate.read(b, off, len); } + @Override public int available() throws IOException { return delegate.available(); } + @Override public void close() throws IOException { delegate.close(); } + @Override public synchronized void mark(int readlimit) { delegate.mark(readlimit); } + @Override public synchronized void reset() throws IOException { delegate.reset(); } + @Override public boolean markSupported() { return delegate.markSupported(); } } \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/SmartFileUtils.java b/core/src/main/java/pl/skidam/automodpack_core/utils/SmartFileUtils.java index e9468d596..787bce52d 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/SmartFileUtils.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/SmartFileUtils.java @@ -1,12 +1,9 @@ package pl.skidam.automodpack_core.utils; import java.io.*; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.security.MessageDigest; -import java.util.*; +import java.nio.channels.FileChannel; +import java.nio.file.*; +import java.util.Arrays; import java.util.stream.Stream; import static pl.skidam.automodpack_core.Constants.*; @@ -15,6 +12,8 @@ public class SmartFileUtils { public static final Path CWD = Path.of(System.getProperty("user.dir")); + // --- File Operations (Delete / Copy / Move) --- + public static void executeOrder66(Path file) { executeOrder66(file, true); } @@ -22,8 +21,7 @@ public static void executeOrder66(Path file) { public static void executeOrder66(Path file, boolean saveDummyFiles) { try { Files.deleteIfExists(file); - } catch (IOException ignored) { - } + } catch (IOException ignored) { } if (Files.isRegularFile(file)) { LegacyClientCacheUtils.dummyIT(file); @@ -33,238 +31,132 @@ public static void executeOrder66(Path file, boolean saveDummyFiles) { } } - public static Path getPathFromCWD(String path) { - return getPath(CWD, path); - } - - // Special for use instead of normal resolve, since it wont work because of the leading slash in file - public static Path getPath(Path origin, String path) { - if (origin == null) { - throw new IllegalArgumentException("Origin path must not be null"); - } - if (path == null || path.isBlank()) { - return origin; - } - - path = path.replace('\\', '/'); - - if (path.startsWith("/")) { - path = path.substring(1); + public static void hardlinkFile(Path sourceFile, Path targetFile) throws IOException { + createParentDirs(targetFile); + try { + Files.createLink(targetFile, sourceFile); + } catch (IOException e) { + LOGGER.warn("Failed to create hardlink from {} to {}, falling back to copy", sourceFile, targetFile, e); + copyFile(sourceFile, targetFile); } - - return origin.resolve(path).normalize(); - } - - public static boolean isFilePhysical(Path path) { - return path.getFileSystem() == FileSystems.getDefault(); } - public static void copyFile(Path source, Path destination) throws IOException { - setupFilePaths(destination); + public static void copyFile(Path sourceFile, Path targetFile) throws IOException { + createParentDirs(targetFile); - try (InputStream is = new LockFreeInputStream(source); - OutputStream os = Files.newOutputStream(destination, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE)) { + // Use a temp file to ensure atomicity at the destination + Path tempTargetFile = targetFile.resolveSibling(targetFile.getFileName() + ".tmp_" + System.nanoTime()); - is.transferTo(os); - } catch (IOException e) { - LOGGER.error("Failed to copy a file from {} to {}", source, destination); + try { + // Copy Source -> Temp + performSmartCopy(sourceFile, tempTargetFile); + // Promote Temp -> Target + moveFile(tempTargetFile, targetFile); + } catch (Exception e) { + try { Files.deleteIfExists(tempTargetFile); } catch (IOException ignored) {} + LOGGER.error("Failed to copy file from {} to {}", sourceFile, targetFile, e); + throw e; } } - public static void setupFilePaths(Path file) throws IOException { - if (!Files.exists(file)) { - if (!Files.exists(file.getParent())) { - Files.createDirectories(file.getParent()); + public static void moveFile(Path sourceFile, Path targetFile) throws IOException { + try { + // Atomic Move: The gold standard for consistency + Files.move(sourceFile, targetFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + try { + // Fallback: Standard Move + Files.move(sourceFile, targetFile, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ex) { + // Last Resort: Copy & Delete (Required for cross-drive moves) + performSmartCopy(sourceFile, targetFile); + Files.deleteIfExists(sourceFile); } - // Windows? #302 -// Files.createFile(destination); - file.toFile().createNewFile(); } } - public static boolean compareSmallFile(Path path, byte[] referenceBytes) { + private static void performSmartCopy(Path source, Path target) throws IOException { try { - if (Files.size(path) != referenceBytes.length) { - return false; - } + // Try Native reflink (CoW) on Java 20+ + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + // Fallback to Zero-Copy Channel Transfer (Handles locked files on Windows) + copyViaChannel(source, target); + } + } - try (InputStream is = new LockFreeInputStream(path)) { - // Java 11+ readNBytes reads exactly X bytes or until EOF. - // Since we know the file is small (~200b), reading it into RAM is perfectly fine. - byte[] fileContent = is.readNBytes(referenceBytes.length); + private static void copyViaChannel(Path sourceFile, Path targetFile) throws IOException { + try (FileChannel source = LockFreeInputStream.openChannel(sourceFile); + FileChannel target = FileChannel.open(targetFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { - // Vectorized Comparison (AVX optimized in Java 17) - return Arrays.equals(fileContent, referenceBytes); + long count = source.size(); + long position = 0; + while (position < count) { + position += source.transferTo(position, count - position, target); } - } catch (Exception e) { - LOGGER.error("Error comparing file: {}", path, e); - return false; } } - // Formats path to be relative to the modpack directory - modpack-content format - // arguments can not be null - public static String formatPath(final Path modpackFile, final Path modpackPath) { - if (modpackPath == null || modpackFile == null) { - throw new IllegalArgumentException("Arguments are null - modpackPath: " + modpackPath + ", modpackFile: " + modpackFile); - } - - final String modpackFileStr = modpackFile.normalize().toString(); - final String modpackFileStrAbs = modpackFile.toAbsolutePath().normalize().toString(); - final String modpackPathStrAbs = modpackPath.toAbsolutePath().normalize().toString(); - final String cwdStrAbs = Path.of(System.getProperty("user.dir")).toAbsolutePath().normalize().toString(); - - String formattedFile = modpackFileStr; + // --- Directory & Path Logic --- - // Checks if in file parents paths (absolute path) there is modpack directory (absolute path) - if (modpackFileStrAbs.startsWith(modpackPathStrAbs)) { - formattedFile = modpackFileStrAbs.substring(modpackPathStrAbs.length()); - } else if (modpackFileStrAbs.startsWith(cwdStrAbs)) { - formattedFile = modpackFileStrAbs.substring(cwdStrAbs.length()); - } else if (!modpackFileStrAbs.equals(modpackFileStr)) { // possible in e.g. docker - LOGGER.error("File: {} ({}) is not in modpack directory: {} ({}) or current working directory: {}", modpackFileStr, modpackFileStrAbs, modpackPath, modpackPathStrAbs, cwdStrAbs); + public static void createParentDirs(Path file) throws IOException { + Path parent = file.getParent(); + if (parent != null && !Files.exists(parent)) { + Files.createDirectories(parent); } - - formattedFile = formattedFile.replace(File.separator, "/"); - - // Its probably useless, but just in case - formattedFile = prefixSlash(formattedFile); - - return formattedFile; } - public static String prefixSlash(String path) { - if (!path.isEmpty() && path.charAt(0) == '/') { - return path; + public static boolean isEmptyDirectory(Path parentPath) throws IOException { + if (!Files.isDirectory(parentPath)) return false; + try (Stream pathStream = Files.list(parentPath)) { + return pathStream.findAny().isEmpty(); } - return "/" + path; } - public static String getHash(Path path) { + public static boolean compareSmallFile(Path path, byte[] referenceBytes) { try { - MessageDigest digest = MessageDigest.getInstance("SHA-1"); - + if (Files.size(path) != referenceBytes.length) return false; try (InputStream is = new LockFreeInputStream(path)) { - byte[] buffer = new byte[64 * 1024]; - int bytesRead; - while ((bytesRead = is.read(buffer)) != -1) { - digest.update(buffer, 0, bytesRead); - } + return Arrays.equals(is.readNBytes(referenceBytes.length), referenceBytes); } - - return HexFormat.of().formatHex(digest.digest()); - } catch (IOException ignored) { // we don't really care about this exception, file may just not exists or be a directory } catch (Exception e) { - LOGGER.error("Failed to get hash for path: {}", path, e); + LOGGER.error("Error comparing file: {}", path, e); + return false; } - return null; } - // We do double pass to avoid storing whole file in memory - public static String getCurseforgeMurmurHash(Path file) throws IOException { - if (!Files.exists(file)) { - return null; - } + public static Path getPathFromCWD(String path) { + return getPath(CWD, path); + } - // MurmurHash2 Constants - final int m = 0x5bd1e995; - final int r = 24; - final int seed = 1; + public static Path getPath(Path origin, String path) { + if (origin == null) throw new IllegalArgumentException("Origin path must not be null"); + if (path == null || path.isBlank()) return origin; - // Pass 1 - // We scan the file just to count non-whitespace bytes - long validLength = 0; + if (path.indexOf('\\') >= 0) path = path.replace('\\', '/'); + if (path.startsWith("/")) path = path.substring(1); - byte[] buffer = new byte[64 * 1024]; + return origin.resolve(path).normalize(); + } - try (InputStream is = new LockFreeInputStream(file)) { - int bytesRead; - while ((bytesRead = is.read(buffer)) != -1) { - for (int i = 0; i < bytesRead; i++) { - byte b = buffer[i]; - // Check for whitespace (Tab, LF, CR, Space) - if (b != 0x9 && b != 0xa && b != 0xd && b != 0x20) { - validLength++; - } - } - } + public static String formatPath(final Path modpackFile, final Path modpackPath) { + if (modpackPath == null || modpackFile == null) { + throw new IllegalArgumentException("Arguments cannot be null"); } - // Pass 2 - // Now we have the length, we can initialize 'h' correctly with Bitwise XOR - long h = (seed ^ validLength); - long k = 0; - int shift = 0; - - try (InputStream is = new LockFreeInputStream(file)) { - int bytesRead; - while ((bytesRead = is.read(buffer)) != -1) { - for (int i = 0; i < bytesRead; i++) { - byte b = buffer[i]; - - // Same filter logic - if (b == 0x9 || b == 0xa || b == 0xd || b == 0x20) { - continue; - } - - // Append byte to current 4-byte chunk 'k' - k = k | ((long) (b & 0xFF) << shift); - shift += 8; - - // If chunk is full (32 bits), mix it into 'h' - if (shift == 32) { - h = 0x00000000FFFFFFFFL & h; - - k = k * m; - k = 0x00000000FFFFFFFFL & k; + String modpackFileStrAbs = modpackFile.toAbsolutePath().normalize().toString(); + String modpackPathStrAbs = modpackPath.toAbsolutePath().normalize().toString(); + String cwdStrAbs = CWD.toAbsolutePath().normalize().toString(); - k = k ^ (k >> r); - k = 0x00000000FFFFFFFFL & k; + String formattedFile = modpackFile.normalize().toString(); - k = k * m; - k = 0x00000000FFFFFFFFL & k; - - h = h * m; - h = 0x00000000FFFFFFFFL & h; - - h = h ^ k; - h = 0x00000000FFFFFFFFL & h; - - // Reset chunk - k = 0; - shift = 0; - } - } - } - } - - if (shift > 0) { - h = h ^ k; - h = 0x00000000FFFFFFFFL & h; - - h = h * m; - h = 0x00000000FFFFFFFFL & h; + if (modpackFileStrAbs.startsWith(modpackPathStrAbs)) { + formattedFile = modpackFileStrAbs.substring(modpackPathStrAbs.length()); + } else if (modpackFileStrAbs.startsWith(cwdStrAbs)) { + formattedFile = modpackFileStrAbs.substring(cwdStrAbs.length()); } - h = h ^ (h >> 13); - h = 0x00000000FFFFFFFFL & h; - - h = h * m; - h = 0x00000000FFFFFFFFL & h; - - h = h ^ (h >> 15); - h = 0x00000000FFFFFFFFL & h; - - return String.valueOf(h); - } - - public static boolean isEmptyDirectory(Path parentPath) throws IOException { - if (!Files.isDirectory(parentPath)) return false; - try (Stream pathStream = Files.list(parentPath)) { - return pathStream.findAny().isEmpty(); - } + formattedFile = formattedFile.replace(File.separator, "/"); + return formattedFile.startsWith("/") ? formattedFile : "/" + formattedFile; } -} +} \ No newline at end of file diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java index 0c45aafb4..afb52f226 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java @@ -2,7 +2,7 @@ import org.h2.mvstore.MVMap; import org.h2.mvstore.MVStore; -import pl.skidam.automodpack_core.utils.SmartFileUtils; +import pl.skidam.automodpack_core.utils.HashUtils; import java.io.IOException; import java.io.Serializable; @@ -85,7 +85,7 @@ public String getOrComputeHash(Path file) throws IOException { } // Actual work happens here - String newHash = SmartFileUtils.getHash(absPath); + String newHash = HashUtils.getHash(absPath); CachedFile newRecord = new CachedFile(newHash, currentTime, currentSize, currentFileKey); fileMetadataMap.put(pathKey, newRecord); diff --git a/core/src/test/java/pl/skidam/automodpack_core/utils/SmartFileUtilsTest.java b/core/src/test/java/pl/skidam/automodpack_core/utils/SmartFileUtilsTest.java index bde3a3d55..05b253ebe 100644 --- a/core/src/test/java/pl/skidam/automodpack_core/utils/SmartFileUtilsTest.java +++ b/core/src/test/java/pl/skidam/automodpack_core/utils/SmartFileUtilsTest.java @@ -21,7 +21,7 @@ void testSHA1Hash_KnownValue() throws IOException, NoSuchAlgorithmException { String content = "test content 2137!"; Files.writeString(file, content); - String actualHash = SmartFileUtils.getHash(file); + String actualHash = HashUtils.getHash(file); assertNotNull(actualHash, "Hash should not be null"); assertEquals("16883d77e42fcb574c70e31cda49b3f955a48be8", actualHash, "getHash should return the correct SHA-1 hash"); @@ -32,7 +32,7 @@ void testMurmurHash_KnownValue() throws IOException { Path file = tempDir.resolve("murmur-test.txt"); Files.writeString(file, "test content 2137!"); - String actualHash = SmartFileUtils.getCurseforgeMurmurHash(file); + String actualHash = HashUtils.getCurseforgeMurmurHash(file); assertEquals("3151456706", actualHash, "MurmurHash for 'test' should match known constant"); } @@ -48,8 +48,8 @@ void testMurmurHash_IgnoresWhitespace() throws IOException { Files.writeString(cleanFile, cleanContent); Files.writeString(messyFile, messyContent); - String cleanHash = SmartFileUtils.getCurseforgeMurmurHash(cleanFile); - String messyHash = SmartFileUtils.getCurseforgeMurmurHash(messyFile); + String cleanHash = HashUtils.getCurseforgeMurmurHash(cleanFile); + String messyHash = HashUtils.getCurseforgeMurmurHash(messyFile); assertEquals(cleanHash, messyHash, "Hashes should be identical despite whitespace differences"); assertEquals("2667173943", messyHash, "Messy file should still hash to the value of 'test'"); @@ -59,7 +59,7 @@ void testMurmurHash_IgnoresWhitespace() throws IOException { void testGetHash_NonExistentFile() { Path missingFile = tempDir.resolve("does-not-exist.txt"); - String result = SmartFileUtils.getHash(missingFile); + String result = HashUtils.getHash(missingFile); assertNull(result, "Should return null for missing file"); } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java index a84ffced6..feab6a00a 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java @@ -1,6 +1,7 @@ package pl.skidam.automodpack_loader_core; import pl.skidam.automodpack_core.config.Jsons; +import pl.skidam.automodpack_core.utils.HashUtils; import pl.skidam.automodpack_core.utils.SmartFileUtils; import pl.skidam.automodpack_core.loader.LoaderManagerService; import pl.skidam.automodpack_core.utils.LockFreeInputStream; @@ -102,7 +103,7 @@ public static boolean update(Jsons.ModpackContentFields serverModpackContent) { } // Exact Hash Match (Fastest check) - if (automodpack.SHA1Hash().equals(SmartFileUtils.getHash(THIS_MOD_JAR))) { + if (automodpack.SHA1Hash().equals(HashUtils.getHash(THIS_MOD_JAR))) { LOGGER.info("Already on the target version (Hash match): {}", AM_VERSION); return false; } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index b0e928838..1d8c2a258 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -179,8 +179,8 @@ public void startUpdate(Set files try (var cache = FileMetadataCache.open(hashCacheDBFile)) { // Don't download files which already exist - // The existing files will be copied over in the applyModpack method - var finalFilesToUpdate = ModpackUtils.getOnlyNonExistingFiles(filesToUpdate, cache); + ModpackUtils.populateStoreFromCWD(filesToUpdate, cache); + var finalFilesToUpdate = ModpackUtils.identifyUncachedFiles(filesToUpdate); // Rename modpack modpackDir = ModpackUtils.renameModpackDir(serverModpackContent, modpackDir); @@ -441,6 +441,8 @@ private boolean applyModpack(FileMetadataCache cache) throws Exception { throw new IllegalStateException("Failed to load modpack content"); // Something gone very wrong... } + ModpackUtils.hardlinkModpack(modpackDir, modpackContent, cache); + // Prepare modpack, analyze nested mods List conflictingNestedMods = MODPACK_LOADER.getModpackNestedConflicts(modpackDir, cache); @@ -450,8 +452,6 @@ private boolean applyModpack(FileMetadataCache cache) throws Exception { Set workaroundMods = new WorkaroundUtil(modpackDir).getWorkaroundMods(modpackContent); Set filesNotToCopy = getFilesNotToCopy(modpackContent.list, workaroundMods); boolean needsRestart1 = ModpackUtils.correctFilesLocations(modpackDir, modpackContent, filesNotToCopy, cache); - workaroundMods = new WorkaroundUtil(modpackDir).getWorkaroundMods(modpackContent); - filesNotToCopy = getFilesNotToCopy(modpackContent.list, workaroundMods); Set modpackMods = new HashSet<>(); Collection modpackModList = new ArrayList<>(); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java index e202a7e86..eda68bbcb 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java @@ -108,36 +108,65 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac return new UpdateCheckResult(false, Set.of()); } - // TODO check more dirs, preferably with a database of all indexed files by their hashes - // This will scan the CWD with CustomFileUtils.getPathFromCWD(modpack.file) and if the file exist with matching hash, it won't be returned in the final set - // Used to avoid downloading files that are already present and valid on disk so we can just copy them over instead of downloading them all again - public static Set getOnlyNonExistingFiles(Set filesToCheck, FileMetadataCache cache) { - Set nonExistingFiles = new HashSet<>(); + // Scans for files missing from the store. If found in the CWD (and the hash matches), copies them to the store. + public static void populateStoreFromCWD(Set filesToUpdate, FileMetadataCache cache) { + for (var entry : filesToUpdate) { + Path storeFile = SmartFileUtils.getPath(storeDir, entry.sha1); - LOGGER.info("Checking for existing files to skip downloading..."); - var time = System.currentTimeMillis(); - int filesSkipped = 0; + if (Files.exists(storeFile)) { + LOGGER.debug("File already exists in store: {}", entry.file); + continue; + } - for (var entry : filesToCheck) { Path fileInCWD = SmartFileUtils.getPathFromCWD(entry.file); if (Files.isRegularFile(fileInCWD)) { String diskHash = cache.getHashOrNull(fileInCWD); if (diskHash.equalsIgnoreCase(entry.sha1)) { - LOGGER.debug("File already exists and matches hash, skipping download: {}", entry.file); - filesSkipped++; - continue; + LOGGER.info("Copying existing file from CWD to store: {}", entry.file); + try { + SmartFileUtils.copyFile(fileInCWD, storeFile); + } catch (IOException e) { + LOGGER.error("Failed to copy file from CWD to store: {}", entry.file, e); + } } } - - nonExistingFiles.add(entry); } + } + // Returns the set of files that are missing from the store. + public static Set identifyUncachedFiles(Set filesToCheck) { + Set nonExistingFiles = new HashSet<>(); + for (var entry : filesToCheck) { + Path storeFile = SmartFileUtils.getPath(storeDir, entry.sha1); - LOGGER.info("Finished checking for existing files in CWD, {} files left to download (skipped {} existing). Took {} ms", nonExistingFiles.size(), filesSkipped, System.currentTimeMillis() - time); - + if (!Files.exists(storeFile)) { + nonExistingFiles.add(entry); + } + } return nonExistingFiles; } + // Installs files from the store (storeDir/) to the instance (modpackDir/). + // Attempts to hardlink first, falls back to a copy if that fails. + public static void hardlinkModpack(Path modpackDir, Jsons.ModpackContentFields serverModpackContent, FileMetadataCache cache) throws IOException { + for (Jsons.ModpackContentFields.ModpackContentItem contentItem : serverModpackContent.list) { + String formattedFile = contentItem.file; + Path modpackFile = SmartFileUtils.getPath(modpackDir, formattedFile); + Path storeFile = SmartFileUtils.getPath(storeDir, contentItem.sha1); + + if (!Files.exists(modpackFile)) { + LOGGER.debug("Hard-linking {} file to the modpack directory", formattedFile); + SmartFileUtils.hardlinkFile(storeFile, modpackFile); + } else { + String modpackFileHash = cache.getHashOrNull(modpackFile); + if (!contentItem.sha1.equalsIgnoreCase(modpackFileHash)) { + LOGGER.debug("Over-hard-linking {} file in the modpack directory", formattedFile); + SmartFileUtils.hardlinkFile(storeFile, modpackFile); + } + } + } + } + public static boolean deleteFilesMarkedForDeletionByTheServer(Set filesToDeleteOnClient, FileMetadataCache cache) { if (!clientConfig.allowRemoteNonModpackDeletions) { if (!filesToDeleteOnClient.isEmpty()) { @@ -230,12 +259,6 @@ public static boolean correctFilesLocations(Path modpackDir, Jsons.ModpackConten boolean runFileHashMatch = false; if (runFileExists) runFileHashMatch = Objects.equals(contentItem.sha1, cache.getHashOrNull(runFile)); - if (runFileHashMatch && !modpackFileExists) { - LOGGER.debug("Copying {} file to the modpack directory", formattedFile); - SmartFileUtils.copyFile(runFile, modpackFile); - modpackFileExists = true; - } - // We only copy mods to the run directory which are not ignored - which need a workaround // If its any other file type, always copy if (filesNotToCopy.contains(formattedFile)) { diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java index 9a10646c2..784f184e1 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java @@ -2,6 +2,7 @@ import pl.skidam.automodpack_core.protocol.NetUtils; import pl.skidam.automodpack_core.utils.SmartFileUtils; +import pl.skidam.automodpack_core.utils.HashUtils; import pl.skidam.automodpack_core.utils.CustomThreadFactoryBuilder; import pl.skidam.automodpack_core.utils.FileInspection; import pl.skidam.automodpack_core.protocol.DownloadClient; @@ -18,184 +19,239 @@ import static pl.skidam.automodpack_core.Constants.*; public class DownloadManager { + private static final int MAX_DOWNLOADS_IN_PROGRESS = 5; - private static final int MAX_DOWNLOAD_ATTEMPTS = 2; // its actually 3, but we start from 0 - private final ExecutorService DOWNLOAD_EXECUTOR = Executors.newFixedThreadPool(MAX_DOWNLOADS_IN_PROGRESS, new CustomThreadFactoryBuilder().setNameFormat("AutoModpackDownload-%d").build()); + // Actually 3 attempts (0, 1, 2) + private static final int MAX_DOWNLOAD_ATTEMPTS = 2; + + private final ExecutorService downloadExecutor = Executors.newFixedThreadPool( + MAX_DOWNLOADS_IN_PROGRESS, + new CustomThreadFactoryBuilder().setNameFormat("AutoModpackDownload-%d").build() + ); + private DownloadClient downloadClient = null; - private boolean cancelled = false; + private volatile boolean cancelled = false; + + // TODO remove it, we should assume that if hash matches file is the same so we don't need to separately track path+hash pairs + // Maps a specific file request (Path + Hash) to a download task private final Map queuedDownloads = new ConcurrentHashMap<>(); public final Map downloadsInProgress = new ConcurrentHashMap<>(); + private long bytesDownloaded = 0; private long bytesToDownload = 0; private int addedToQueue = 0; private int downloaded = 0; + private final Semaphore semaphore = new Semaphore(0); private final SpeedMeter speedMeter = new SpeedMeter(this); + public DownloadManager() { } + public DownloadManager(long bytesToDownload) { this.bytesToDownload = bytesToDownload; } - // TODO: make caching system which detects if the same file was downloaded before and if so copy it instead of downloading again public void attachDownloadClient(DownloadClient downloadClient) { this.downloadClient = downloadClient; } + /** + * Queues a file for download. + * The file will be downloaded to the global store (by hash) and then copied to the 'file' path. + */ + // TODO dont copy (update self updater to either use store directly or write a method to directly download without store) public void download(Path file, String sha1, List urls, Runnable successCallback, Runnable failureCallback) { FileInspection.HashPathPair hashPathPair = new FileInspection.HashPathPair(sha1, file); + if (queuedDownloads.containsKey(hashPathPair)) return; + queuedDownloads.put(hashPathPair, new QueuedDownload(file, urls, 0, successCallback, failureCallback)); addedToQueue++; + downloadNext(); } - private void downloadTask(FileInspection.HashPathPair hashPathPair, QueuedDownload queuedDownload) throws Exception { - LOGGER.info("Downloading {} - {}", queuedDownload.file.getFileName(), queuedDownload.urls); - - int numberOfIndexes = queuedDownload.urls.size(); - int urlIndex = Math.min(queuedDownload.attempts / MAX_DOWNLOAD_ATTEMPTS, numberOfIndexes); - String url = "host"; - if (queuedDownload.urls.size() > urlIndex) { // avoids IndexOutOfBoundsException - url = queuedDownload.urls.get(urlIndex); + private synchronized void downloadNext() { + if (downloadsInProgress.size() >= MAX_DOWNLOADS_IN_PROGRESS || queuedDownloads.isEmpty()) { + return; } + + var entry = queuedDownloads.entrySet().iterator().next(); + FileInspection.HashPathPair key = entry.getKey(); + QueuedDownload task = queuedDownloads.remove(key); + + if (task == null) return; + + CompletableFuture future = CompletableFuture.runAsync(() -> { + try { + processDownloadTask(key, task); + } catch (Exception e) { + LOGGER.error("Fatal error executing download task for {}", task.file.getFileName(), e); + } + }, downloadExecutor); + + downloadsInProgress.put(key, new DownloadData(future, task.file)); + } + + /** + * Main logic for processing a single download request. + */ + private void processDownloadTask(FileInspection.HashPathPair hashPathPair, QueuedDownload task) { + LOGGER.info("Processing {} - Hash: {}", task.file.getFileName(), hashPathPair.hash()); + + Path storeFile = storeDir.resolve(hashPathPair.hash()); + boolean success = false; boolean interrupted = false; try { - if (url != null && !Objects.equals(url, "host") && queuedDownload.attempts < MAX_DOWNLOAD_ATTEMPTS * numberOfIndexes) { - httpDownloadFile(url, hashPathPair, queuedDownload); - } else if (downloadClient != null) { - hostDownloadFile(hashPathPair, queuedDownload); + // Check if file already exists in Store + if (verifyFile(storeFile, hashPathPair.hash())) { + // Increment progress for the cached file so percentage calc remains accurate + long size = Files.size(storeFile); + bytesDownloaded += size; + // Don't update speedMeter for cache hits to avoid fake speed spikes + success = true; } else { - LOGGER.error("No download client attached, can't download file - {}", queuedDownload.file.getFileName()); + // Not in store, attempt download + success = attemptDownload(hashPathPair, task, storeFile); } + } catch (InterruptedException e) { interrupted = true; - } catch (SocketTimeoutException e) { - LOGGER.warn("Timeout - {} - {} - {}", queuedDownload.file, e, e.fillInStackTrace()); } catch (Exception e) { - LOGGER.warn("Error while downloading file - {} - {} - {}", queuedDownload.file, e, e.fillInStackTrace()); + LOGGER.warn("Unexpected error processing {}", task.file, e); } finally { - downloadsInProgress.remove(hashPathPair); - boolean failed = true; - - if (Files.exists(queuedDownload.file)) { - String hash = SmartFileUtils.getHash(queuedDownload.file); - - if (Objects.equals(hash, hashPathPair.hash())) { - // Runs on success - failed = false; - downloaded++; - LOGGER.info("Successfully downloaded {} from {}", queuedDownload.file.getFileName(), url); - queuedDownload.successCallback.run(); - semaphore.release(); - } - } + cleanupAndFinalize(hashPathPair, task, storeFile, success, interrupted); + } + } + + /** + * Tries to download the file from available sources (URLs or Client). + * Returns true if downloaded and verified successfully. + */ + private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, QueuedDownload task, Path storeFile) throws InterruptedException { + int numberOfIndexes = task.urls.size(); + // Determine which URL to try based on retry count + int urlIndex = Math.min(task.attempts / MAX_DOWNLOAD_ATTEMPTS, numberOfIndexes); - if (failed) { - bytesToDownload += queuedDownload.file.toFile().length(); // Add size of the whole file again because we will try to download it again - SmartFileUtils.executeOrder66(queuedDownload.file); - - if (!interrupted) { - if (queuedDownload.attempts < (numberOfIndexes + 1) * MAX_DOWNLOAD_ATTEMPTS) { - LOGGER.warn("Download of {} failed, retrying!", queuedDownload.file.getFileName()); - queuedDownload.attempts++; - queuedDownloads.put(hashPathPair, queuedDownload); - } else { - LOGGER.error("Download of {} failed!", queuedDownload.file.getFileName()); - queuedDownload.failureCallback.run(); - semaphore.release(); - } - } + String url = (task.urls.size() > urlIndex) ? task.urls.get(urlIndex) : null; + + // Use a temporary file for downloading to ensure atomicity + Path tempStoreFile = storeDir.resolve(hashPathPair.hash() + ".tmp"); + + try { + if (url != null && !Objects.equals(url, "host") && task.attempts < MAX_DOWNLOAD_ATTEMPTS * numberOfIndexes) { + httpDownloadFile(url, tempStoreFile); + } else if (downloadClient != null) { + hostDownloadFile(hashPathPair, tempStoreFile); + } else { + LOGGER.error("No valid source found for {}", task.file.getFileName()); + return false; } - if (!interrupted) { - downloadNext(); + // Verify the temp file + if (verifyFile(tempStoreFile, hashPathPair.hash())) { + // Move temp file to actual store file + SmartFileUtils.moveFile(tempStoreFile, storeFile); + return true; + } else { + LOGGER.warn("Hash mismatch for downloaded file {}", task.file.getFileName()); + SmartFileUtils.executeOrder66(tempStoreFile); + return false; } + } catch (IOException e) { + LOGGER.warn("Download I/O error for {}: {}", task.file.getFileName(), e.getMessage()); + SmartFileUtils.executeOrder66(tempStoreFile); + return false; } } - private synchronized void downloadNext() { - if (downloadsInProgress.size() < MAX_DOWNLOADS_IN_PROGRESS && !queuedDownloads.isEmpty()) { - FileInspection.HashPathPair hashAndPath = queuedDownloads.keySet().stream().findFirst().get(); - QueuedDownload queuedDownload = queuedDownloads.remove(hashAndPath); - - if (queuedDownload == null) { - return; + /** + * Handles the cleanup, callbacks, and retries. + */ + private void cleanupAndFinalize(FileInspection.HashPathPair key, QueuedDownload task, Path storeFile, boolean success, boolean interrupted) { + downloadsInProgress.remove(key); + + if (success) { + try { + // Copy from Store -> Destination + SmartFileUtils.copyFile(storeFile, task.file); + + downloaded++; + LOGGER.info("Finished: {} -> {}", storeFile.getFileName(), task.file.getFileName()); + task.successCallback.run(); + semaphore.release(); + } catch (IOException e) { + LOGGER.error("Failed to copy from store to destination: {}", task.file, e); + // Technically a failure in the final step, treat as retry-able + handleRetry(key, task, interrupted); } + } else { + handleRetry(key, task, interrupted); + } - CompletableFuture future = CompletableFuture.runAsync(() -> { - try { - downloadTask(hashAndPath, queuedDownload); - } catch (Exception e) { - LOGGER.error("Error while downloading file - {}", queuedDownload.file.getFileName(), e); - } - }, DOWNLOAD_EXECUTOR); - - downloadsInProgress.put(hashAndPath, new DownloadData(future, queuedDownload.file)); + if (!interrupted) { + downloadNext(); } } - private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, QueuedDownload queuedDownload) throws IOException, InterruptedException { - Path outFile = queuedDownload.file; + private void handleRetry(FileInspection.HashPathPair key, QueuedDownload task, boolean interrupted) { + if (interrupted) return; - if (Files.exists(outFile)) { - if (Objects.equals(hashPathPair.hash(), SmartFileUtils.getHash(outFile))) { - return; - } else { - SmartFileUtils.executeOrder66(outFile); - } + // Increase total bytes because we will try to download this amount again + try { + // Estimate size if possible, or just ignore (original code logic) + if (Files.exists(task.file)) bytesToDownload += Files.size(task.file); + } catch (IOException ignored) {} + + // Wipe destination just in case + SmartFileUtils.executeOrder66(task.file); + + int numberOfIndexes = task.urls.size(); + int maxAttempts = (numberOfIndexes + 1) * MAX_DOWNLOAD_ATTEMPTS; + + if (task.attempts < maxAttempts) { + LOGGER.warn("Retrying download: {}", task.file.getFileName()); + task.attempts++; + queuedDownloads.put(key, task); + } else { + LOGGER.error("Permanently failed to download: {}", task.file.getFileName()); + task.failureCallback.run(); + semaphore.release(); } + } - SmartFileUtils.setupFilePaths(outFile); + // --- Download Implementations --- - var future = downloadClient.downloadFile(hashPathPair.hash().getBytes(StandardCharsets.UTF_8), outFile, (bytes) -> { - bytesDownloaded += bytes; - speedMeter.addDownloadedBytes(bytes); - }); + private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, Path targetFile) throws IOException { + SmartFileUtils.createParentDirs(targetFile); + var future = downloadClient.downloadFile(hashPathPair.hash().getBytes(StandardCharsets.UTF_8), targetFile, this::updateProgress); future.join(); } - private void httpDownloadFile(String url, FileInspection.HashPathPair hashPathPair, QueuedDownload queuedDownload) throws IOException, InterruptedException { + private void httpDownloadFile(String urlString, Path targetFile) throws IOException, InterruptedException { + SmartFileUtils.createParentDirs(targetFile); + LOGGER.info("Downloading from {}", urlString); - Path outFile = queuedDownload.file; + URLConnection connection = getHttpConnection(urlString); - if (Files.exists(outFile)) { - if (Objects.equals(hashPathPair.hash(), SmartFileUtils.getHash(outFile))) { - return; - } else { - SmartFileUtils.executeOrder66(outFile); - } - } - - SmartFileUtils.setupFilePaths(outFile); - - URLConnection connection = getHttpConnection(url); - - InputStream rawInputStream = connection.getInputStream(); - InputStream inputStream = ("gzip".equals(connection.getHeaderField("Content-Encoding"))) ? new GZIPInputStream(rawInputStream) : rawInputStream; - - try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outFile.toFile()), 64 * 1024); - InputStream is = inputStream) { + try (InputStream rawIn = connection.getInputStream(); + InputStream in = "gzip".equals(connection.getHeaderField("Content-Encoding")) ? new GZIPInputStream(rawIn) : rawIn; + OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile.toFile()), 64 * 1024)) { byte[] buffer = new byte[NetUtils.DEFAULT_CHUNK_SIZE]; int bytesRead; - while ((bytesRead = is.read(buffer)) != -1) { - bytesDownloaded += bytesRead; - speedMeter.addDownloadedBytes(bytesRead); - outputStream.write(buffer, 0, bytesRead); - - if (Thread.currentThread().isInterrupted()) { - throw new InterruptedException("Download got cancelled"); - } + while ((bytesRead = in.read(buffer)) != -1) { + if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Download cancelled"); + out.write(buffer, 0, bytesRead); + updateProgress(bytesRead); } } } - private URLConnection getHttpConnection(String url) throws IOException { - LOGGER.info("Downloading from {}", url); - - URL connectionUrl = new URL(url); - URLConnection connection = connectionUrl.openConnection(); + private URLConnection getHttpConnection(String urlString) throws IOException { + URL url = new URL(urlString); + URLConnection connection = url.openConnection(); connection.addRequestProperty("Accept-Encoding", "gzip"); connection.addRequestProperty("User-Agent", "github/skidamek/automodpack/" + AM_VERSION); connection.setConnectTimeout(10000); @@ -203,69 +259,64 @@ private URLConnection getHttpConnection(String url) throws IOException { return connection; } + private void updateProgress(long bytes) { + bytesDownloaded += bytes; + speedMeter.addDownloadedBytes(bytes); + } - public void joinAll() throws InterruptedException { - semaphore.acquire(addedToQueue); + // --- Helpers --- - // Means that download got cancelled, throw exception to don't finish the modpack updater logic - if (DOWNLOAD_EXECUTOR.isShutdown()) { - throw new InterruptedException(); + private boolean verifyFile(Path file, String expectedHash) { + if (!Files.exists(file)) return false; + try { + return Objects.equals(HashUtils.getHash(file), expectedHash); + } catch (Exception e) { + return false; } + } + + // --- State Management (Preserved) --- + public void joinAll() throws InterruptedException { + semaphore.acquire(addedToQueue); + if (downloadExecutor.isShutdown()) throw new InterruptedException(); semaphore.release(addedToQueue); } - public SpeedMeter getSpeedMeter() { - return speedMeter; - } + public SpeedMeter getSpeedMeter() { return speedMeter; } - public long getTotalBytesRemaining() { - return bytesToDownload - bytesDownloaded; - } + public long getTotalBytesRemaining() { return bytesToDownload - bytesDownloaded; } public int getTotalPercentageOfFileSizeDownloaded() { - if (bytesDownloaded == 0 || bytesToDownload == 0) { - return 0; - } - + if (bytesToDownload == 0) return 0; int percentage = (int) (bytesDownloaded * 100 / bytesToDownload); return Math.max(0, Math.min(100, percentage)); } - public String getStage() { - // files downloaded / files downloaded + queued - return downloaded + "/" + addedToQueue; - } + public String getStage() { return downloaded + "/" + addedToQueue; } - public boolean isRunning() { - return !DOWNLOAD_EXECUTOR.isShutdown(); - } + public boolean isRunning() { return !downloadExecutor.isShutdown(); } - public boolean isCanceled() { - return cancelled; - } + public boolean isCanceled() { return cancelled; } public void cancelAllAndShutdown() { cancelled = true; queuedDownloads.clear(); - downloadsInProgress.forEach((url, downloadData) -> { - downloadData.future.cancel(true); - SmartFileUtils.executeOrder66(downloadData.file); + downloadsInProgress.forEach((key, data) -> { + data.future.cancel(true); + SmartFileUtils.executeOrder66(data.file); }); - // TODO Release the number of occupied permits, not all semaphore.release(addedToQueue); downloadsInProgress.clear(); downloaded = 0; addedToQueue = 0; - if (downloadClient != null) { - downloadClient.close(); - } - - DOWNLOAD_EXECUTOR.shutdown(); + if (downloadClient != null) downloadClient.close(); + downloadExecutor.shutdown(); } + // --- Inner Classes --- public static class QueuedDownload { private final Path file; @@ -273,6 +324,7 @@ public static class QueuedDownload { private int attempts; private final Runnable successCallback; private final Runnable failureCallback; + public QueuedDownload(Path file, List urls, int attempts, Runnable successCallback, Runnable failureCallback) { this.file = file; this.urls = urls; From a74bee956acd91cb80bcdaa09379f5f440d60166 Mon Sep 17 00:00:00 2001 From: skidam Date: Fri, 30 Jan 2026 19:16:20 +0100 Subject: [PATCH 17/22] Make sure the file exists before trying to link from it --- .../skidam/automodpack_loader_core/client/ModpackUtils.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java index eda68bbcb..84527f1b7 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java @@ -154,6 +154,11 @@ public static void hardlinkModpack(Path modpackDir, Jsons.ModpackContentFields s Path modpackFile = SmartFileUtils.getPath(modpackDir, formattedFile); Path storeFile = SmartFileUtils.getPath(storeDir, contentItem.sha1); + if (!Files.exists(storeFile)) { + LOGGER.debug("File {} not found in store, can't hardlink", formattedFile); + return; + } + if (!Files.exists(modpackFile)) { LOGGER.debug("Hard-linking {} file to the modpack directory", formattedFile); SmartFileUtils.hardlinkFile(storeFile, modpackFile); From 710551fb348a613d0f741baef614c040e429a255 Mon Sep 17 00:00:00 2001 From: skidam Date: Sun, 1 Feb 2026 17:29:12 +0100 Subject: [PATCH 18/22] Improve isUpdate performance --- .../utils/cache/FileMetadataCache.java | 15 ++- .../client/ModpackUtils.java | 107 +++++++++++++----- 2 files changed, 95 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java index afb52f226..32e1370ae 100644 --- a/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java +++ b/core/src/main/java/pl/skidam/automodpack_core/utils/cache/FileMetadataCache.java @@ -61,10 +61,14 @@ private FileMetadataCache(Path dbPath) { } public String getOrComputeHash(Path file) throws IOException { + BasicFileAttributes attrs = Files.readAttributes(file, BasicFileAttributes.class); + return getOrComputeHashWithAttributes(file, attrs); + } + + public String getOrComputeHashWithAttributes(Path file, BasicFileAttributes attrs) { Path absPath = file.toAbsolutePath().normalize(); String pathKey = absPath.toString(); - BasicFileAttributes attrs = Files.readAttributes(absPath, BasicFileAttributes.class); long currentSize = attrs.size(); long currentTime = attrs.lastModifiedTime().toMillis(); String currentFileKey = attrs.fileKey() != null ? attrs.fileKey().toString() : "null"; @@ -107,6 +111,15 @@ public String getHashOrNull(Path path) { } } + public String getHashOrNullWithAttributes(Path path, BasicFileAttributes attrs) { + try { + return getOrComputeHashWithAttributes(path, attrs); + } catch (Exception e) { + LOGGER.error("Failed to compute hash for path: {}", path, e); + return null; + } + } + public boolean fastHashCompare(Path file1, Path file2) throws IOException { if (!Files.exists(file1) || !Files.exists(file2)) return false; diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java index 84527f1b7..73151268c 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUtils.java @@ -1,5 +1,6 @@ package pl.skidam.automodpack_loader_core.client; +import org.jetbrains.annotations.NotNull; import pl.skidam.automodpack_core.auth.Secrets; import pl.skidam.automodpack_core.config.ConfigTools; import pl.skidam.automodpack_core.config.Jsons; @@ -16,6 +17,7 @@ import java.io.*; import java.net.*; import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; import java.security.cert.CertificateEncodingException; import java.security.cert.X509Certificate; import java.util.*; @@ -50,40 +52,93 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac return new UpdateCheckResult(true, serverModpackContent.list); } - LOGGER.info("Indexing file system..."); - var start = System.currentTimeMillis(); - - Set existingFileTree; - try (var stream = Files.walk(modpackDir)) { - existingFileTree = stream.collect(Collectors.toSet()); - } catch (IOException e) { - LOGGER.error("Failed to walk directory", e); - return new UpdateCheckResult(true, serverModpackContent.list); - } - LOGGER.info("Verifying content against server list..."); + var start = System.currentTimeMillis(); Set filesToUpdate = ConcurrentHashMap.newKeySet(); + // Group & Sort Server Files (Optimizes Disk Seek Pattern) + // Grouping by parent folder ensures we process the disk sequentially (Dir A, then Dir B). + // TreeMap ensures alphabetical order of directories (HDD friendly). + Map> itemsByDir = + serverModpackContent.list.stream() + .collect(Collectors.groupingBy( + item -> SmartFileUtils.getPath(modpackDir, item.file).getParent(), + TreeMap::new, + Collectors.toList() + )); + try (var cache = FileMetadataCache.open(hashCacheDBFile)) { - serverModpackContent.list.forEach(serverItem -> { - Path serverItemPath = SmartFileUtils.getPath(modpackDir, serverItem.file); - if (!existingFileTree.contains(serverItemPath)) { - filesToUpdate.add(serverItem); // File is missing - return; - } else if (serverItem.editable) { // TODO check if this is enough of a check, what if user already had a file but there's provided the same by a new modpack version which wasnt in the modpack before? - LOGGER.debug("Skipping editable file hash check: {}", serverItem.file); - return; + + // Process Directory by Directory + for (Map.Entry> entry : itemsByDir.entrySet()) { + Path parentDir = entry.getKey(); + List itemsInDir = entry.getValue(); + + // If directory is missing, all items in it are missing. + if (!Files.exists(parentDir)) { + filesToUpdate.addAll(itemsInDir); + continue; } - String hash = cache.getHashOrNull(serverItemPath); - if (hash != null && hash.equals(serverItem.sha1)) { - return; // File is up to date + // Read all file attributes in this folder in ONE pass. + // This map will hold "FileName" -> "Attributes" + Map diskFiles = new HashMap<>(); + + try { + // walkFileTree with depth 1 is efficient on Windows (gets attributes for free within a single syscall) + Files.walkFileTree(parentDir, EnumSet.noneOf(FileVisitOption.class), 1, new SimpleFileVisitor<>() { + @NotNull @Override + public FileVisitResult visitFile(@NotNull Path file, @NotNull BasicFileAttributes attrs) { + diskFiles.put(file.getFileName().toString(), attrs); + return FileVisitResult.CONTINUE; + } + + @NotNull @Override + public FileVisitResult visitFileFailed(@NotNull Path file, @NotNull IOException exc) { + return FileVisitResult.CONTINUE; // Handle locked files or permission errors gracefully + } + }); + } catch (IOException e) { + LOGGER.warn("Failed to inspect directory: {}", parentDir, e); + filesToUpdate.addAll(itemsInDir); + continue; } - // This file needs to be updated - filesToUpdate.add(serverItem); - }); + // Check Individual Files in a given directory (Pure RAM logic, 0 IO) + for (var serverItem : itemsInDir) { + String fileName = Paths.get(serverItem.file).getFileName().toString(); + BasicFileAttributes diskAttrs = diskFiles.get(fileName); + + if (diskAttrs == null) { + // File does not exist in the directory map + filesToUpdate.add(serverItem); + } else { + if (serverItem.editable) { // TODO check if this is enough of a check, what if user already had a file but there's provided the same by a new modpack version which wasn't in the modpack before? + LOGGER.debug("Skipping editable file hash check: {}", serverItem.file); + continue; + } + + // Check Size first from already read attributes + if (diskAttrs.size() != Long.parseLong(serverItem.size)) { + filesToUpdate.add(serverItem); + continue; + } + + // Finally, check Hash + // We pass 'diskAttrs' to the cache so it doesn't need to re-stat the file. + String hash = cache.getHashOrNullWithAttributes(parentDir.resolve(fileName), diskAttrs); + + if (!serverItem.sha1.equalsIgnoreCase(hash)) { + filesToUpdate.add(serverItem); + } + } + } + } + } catch (Exception e) { + LOGGER.error("Error during update check", e); + // Fail-safe: assume update needed if process crashes + return new UpdateCheckResult(true, serverModpackContent.list); } if (!filesToUpdate.isEmpty()) { @@ -99,7 +154,7 @@ public static UpdateCheckResult isUpdate(Jsons.ModpackContentFields serverModpac for (Jsons.ModpackContentFields.ModpackContentItem clientItem : clientModpackContent.list) { if (!serverFileSet.contains(clientItem.file)) { - LOGGER.info("Found file marked for deletion (its no longer on server): {}", clientItem.file); + LOGGER.info("Found file marked for deletion: {}", clientItem.file); return new UpdateCheckResult(true, Set.of()); } } From 3072fa87240f471347367ead2259eae2edc73328 Mon Sep 17 00:00:00 2001 From: skidam Date: Mon, 9 Feb 2026 13:54:32 +0100 Subject: [PATCH 19/22] Rewrite speedometer to make metrics smoother --- .../client/ModpackUpdater.java | 4 +- .../utils/DownloadManager.java | 113 +++---- .../utils/SpeedFormatter.java | 41 +++ .../utils/SpeedMeter.java | 85 ----- .../utils/Speedometer.java | 91 +++++ .../automodpack/client/ui/DownloadScreen.java | 310 ++++++------------ 6 files changed, 292 insertions(+), 352 deletions(-) create mode 100644 loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java delete mode 100644 loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedMeter.java create mode 100644 loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index 1d8c2a258..d9b93114a 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -320,7 +320,7 @@ private void downloadModpack(Set LOGGER.info("Finished downloading files in {}ms", System.currentTimeMillis() - startFetching); - if (downloadManager.isCanceled()) { + if (downloadManager.isCancelled()) { LOGGER.warn("Download canceled"); return; } @@ -415,7 +415,7 @@ private void downloadModpack(Set downloadManager.joinAll(); - if (downloadManager.isCanceled()) { + if (downloadManager.isCancelled()) { LOGGER.warn("Download canceled"); return; } diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java index 784f184e1..31b3e8f17 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java @@ -14,6 +14,7 @@ import java.nio.file.Path; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; import java.util.zip.GZIPInputStream; import static pl.skidam.automodpack_core.Constants.*; @@ -21,7 +22,6 @@ public class DownloadManager { private static final int MAX_DOWNLOADS_IN_PROGRESS = 5; - // Actually 3 attempts (0, 1, 2) private static final int MAX_DOWNLOAD_ATTEMPTS = 2; private final ExecutorService downloadExecutor = Executors.newFixedThreadPool( @@ -32,34 +32,30 @@ public class DownloadManager { private DownloadClient downloadClient = null; private volatile boolean cancelled = false; - // TODO remove it, we should assume that if hash matches file is the same so we don't need to separately track path+hash pairs - // Maps a specific file request (Path + Hash) to a download task private final Map queuedDownloads = new ConcurrentHashMap<>(); public final Map downloadsInProgress = new ConcurrentHashMap<>(); - private long bytesDownloaded = 0; - private long bytesToDownload = 0; + // Stats + private final AtomicLong totalBytesDownloaded = new AtomicLong(0); // For Progress Bar (Includes Cache) + private final AtomicLong totalBytesToDownload = new AtomicLong(0); // For Progress Bar private int addedToQueue = 0; - private int downloaded = 0; + private int downloadedCount = 0; private final Semaphore semaphore = new Semaphore(0); - private final SpeedMeter speedMeter = new SpeedMeter(this); + + private final Speedometer speedometer = new Speedometer(); public DownloadManager() { } public DownloadManager(long bytesToDownload) { - this.bytesToDownload = bytesToDownload; + this.totalBytesToDownload.set(bytesToDownload); + this.speedometer.setExpectedBytes(bytesToDownload); } public void attachDownloadClient(DownloadClient downloadClient) { this.downloadClient = downloadClient; } - /** - * Queues a file for download. - * The file will be downloaded to the global store (by hash) and then copied to the 'file' path. - */ - // TODO dont copy (update self updater to either use store directly or write a method to directly download without store) public void download(Path file, String sha1, List urls, Runnable successCallback, Runnable failureCallback) { FileInspection.HashPathPair hashPathPair = new FileInspection.HashPathPair(sha1, file); @@ -93,9 +89,6 @@ private synchronized void downloadNext() { downloadsInProgress.put(key, new DownloadData(future, task.file)); } - /** - * Main logic for processing a single download request. - */ private void processDownloadTask(FileInspection.HashPathPair hashPathPair, QueuedDownload task) { LOGGER.info("Processing {} - Hash: {}", task.file.getFileName(), hashPathPair.hash()); @@ -104,15 +97,16 @@ private void processDownloadTask(FileInspection.HashPathPair hashPathPair, Queue boolean interrupted = false; try { - // Check if file already exists in Store if (verifyFile(storeFile, hashPathPair.hash())) { - // Increment progress for the cached file so percentage calc remains accurate + // CACHE HIT long size = Files.size(storeFile); - bytesDownloaded += size; - // Don't update speedMeter for cache hits to avoid fake speed spikes + totalBytesDownloaded.addAndGet(size); + // IMPORTANT: Do NOT add cached bytes to Speedometer. + // It would fake a massive speed spike. + success = true; } else { - // Not in store, attempt download + // DOWNLOAD REQUIRED success = attemptDownload(hashPathPair, task, storeFile); } @@ -125,18 +119,11 @@ private void processDownloadTask(FileInspection.HashPathPair hashPathPair, Queue } } - /** - * Tries to download the file from available sources (URLs or Client). - * Returns true if downloaded and verified successfully. - */ private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, QueuedDownload task, Path storeFile) throws InterruptedException { int numberOfIndexes = task.urls.size(); - // Determine which URL to try based on retry count int urlIndex = Math.min(task.attempts / MAX_DOWNLOAD_ATTEMPTS, numberOfIndexes); - String url = (task.urls.size() > urlIndex) ? task.urls.get(urlIndex) : null; - // Use a temporary file for downloading to ensure atomicity Path tempStoreFile = storeDir.resolve(hashPathPair.hash() + ".tmp"); try { @@ -149,9 +136,7 @@ private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, Queued return false; } - // Verify the temp file if (verifyFile(tempStoreFile, hashPathPair.hash())) { - // Move temp file to actual store file SmartFileUtils.moveFile(tempStoreFile, storeFile); return true; } else { @@ -166,24 +151,18 @@ private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, Queued } } - /** - * Handles the cleanup, callbacks, and retries. - */ private void cleanupAndFinalize(FileInspection.HashPathPair key, QueuedDownload task, Path storeFile, boolean success, boolean interrupted) { downloadsInProgress.remove(key); if (success) { try { - // Copy from Store -> Destination SmartFileUtils.copyFile(storeFile, task.file); - - downloaded++; + downloadedCount++; LOGGER.info("Finished: {} -> {}", storeFile.getFileName(), task.file.getFileName()); task.successCallback.run(); semaphore.release(); } catch (IOException e) { LOGGER.error("Failed to copy from store to destination: {}", task.file, e); - // Technically a failure in the final step, treat as retry-able handleRetry(key, task, interrupted); } } else { @@ -198,13 +177,15 @@ private void cleanupAndFinalize(FileInspection.HashPathPair key, QueuedDownload private void handleRetry(FileInspection.HashPathPair key, QueuedDownload task, boolean interrupted) { if (interrupted) return; - // Increase total bytes because we will try to download this amount again try { - // Estimate size if possible, or just ignore (original code logic) - if (Files.exists(task.file)) bytesToDownload += Files.size(task.file); + if (Files.exists(task.file)) { + long size = Files.size(task.file); + // If we retry, we expect to download this size again + totalBytesToDownload.addAndGet(size); + speedometer.setExpectedBytes(totalBytesToDownload.get()); + } } catch (IOException ignored) {} - // Wipe destination just in case SmartFileUtils.executeOrder66(task.file); int numberOfIndexes = task.urls.size(); @@ -225,7 +206,7 @@ private void handleRetry(FileInspection.HashPathPair key, QueuedDownload task, b private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, Path targetFile) throws IOException { SmartFileUtils.createParentDirs(targetFile); - var future = downloadClient.downloadFile(hashPathPair.hash().getBytes(StandardCharsets.UTF_8), targetFile, this::updateProgress); + var future = downloadClient.downloadFile(hashPathPair.hash().getBytes(StandardCharsets.UTF_8), targetFile, this::updateNetworkProgress); future.join(); } @@ -244,7 +225,7 @@ private void httpDownloadFile(String urlString, Path targetFile) throws IOExcept while ((bytesRead = in.read(buffer)) != -1) { if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Download cancelled"); out.write(buffer, 0, bytesRead); - updateProgress(bytesRead); + updateNetworkProgress(bytesRead); } } } @@ -259,13 +240,14 @@ private URLConnection getHttpConnection(String urlString) throws IOException { return connection; } - private void updateProgress(long bytes) { - bytesDownloaded += bytes; - speedMeter.addDownloadedBytes(bytes); + /** + * Updates counters ONLY for actual network traffic. + */ + private void updateNetworkProgress(long bytes) { + totalBytesDownloaded.addAndGet(bytes); + speedometer.addBytes(bytes); } - // --- Helpers --- - private boolean verifyFile(Path file, String expectedHash) { if (!Files.exists(file)) return false; try { @@ -275,7 +257,7 @@ private boolean verifyFile(Path file, String expectedHash) { } } - // --- State Management (Preserved) --- + // --- Getters & Control --- public void joinAll() throws InterruptedException { semaphore.acquire(addedToQueue); @@ -283,21 +265,26 @@ public void joinAll() throws InterruptedException { semaphore.release(addedToQueue); } - public SpeedMeter getSpeedMeter() { return speedMeter; } + // --- UI Helpers --- - public long getTotalBytesRemaining() { return bytesToDownload - bytesDownloaded; } + public long getDownloadSpeed() { + return speedometer.getSpeed(); + } - public int getTotalPercentageOfFileSizeDownloaded() { - if (bytesToDownload == 0) return 0; - int percentage = (int) (bytesDownloaded * 100 / bytesToDownload); - return Math.max(0, Math.min(100, percentage)); + public long getETA() { + return speedometer.getETA(); } - public String getStage() { return downloaded + "/" + addedToQueue; } + public double getPrecisePercentage() { + long total = totalBytesToDownload.get(); + if (total == 0) return 0.0; + double pc = (double) totalBytesDownloaded.get() * 100.0 / total; + return Math.max(0.0, Math.min(100.0, pc)); + } - public boolean isRunning() { return !downloadExecutor.isShutdown(); } + public String getStage() { return downloadedCount + "/" + addedToQueue; } - public boolean isCanceled() { return cancelled; } + public boolean isRunning() { return !downloadExecutor.isShutdown(); } public void cancelAllAndShutdown() { cancelled = true; @@ -309,14 +296,20 @@ public void cancelAllAndShutdown() { semaphore.release(addedToQueue); downloadsInProgress.clear(); - downloaded = 0; + downloadedCount = 0; addedToQueue = 0; if (downloadClient != null) downloadClient.close(); downloadExecutor.shutdown(); } - // --- Inner Classes --- + public boolean isCancelled() { + return cancelled; + } + + public void setCancelled(boolean cancelled) { + this.cancelled = cancelled; + } public static class QueuedDownload { private final Path file; diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java new file mode 100644 index 000000000..c70bf268f --- /dev/null +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java @@ -0,0 +1,41 @@ +package pl.skidam.automodpack_loader_core.utils; + +import java.util.Locale; + +public class SpeedFormatter { + + public static String formatSpeed(long bytesPerSec) { + if (bytesPerSec < 0) return "-1"; + if (bytesPerSec < 1024) return bytesPerSec + " B/s"; + + double kb = bytesPerSec / 1024.0; + if (kb < 1024) { + return String.format(Locale.US, "%.1f KB/s", kb); + } + + double mb = kb / 1024.0; + return String.format(Locale.US, "%.1f MB/s", mb); + } + + public static String formatETA(long seconds) { + if (seconds <= 0) return "-1"; + + long days = seconds / 86400; + long hours = (seconds % 86400) / 3600; + long minutes = (seconds % 3600) / 60; + long secs = seconds % 60; + + if (days > 0) { + return String.format("%dd %dh", days, hours); + } else if (hours > 0) { + // e.g. 1h 05m + return String.format("%dh %02dm", hours, minutes); + } else if (minutes > 0) { + // e.g. 05m 12s + return String.format("%02dm %02ds", minutes, secs); + } else { + // e.g. 45s + return String.format("%ds", secs); + } + } +} \ No newline at end of file diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedMeter.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedMeter.java deleted file mode 100644 index 8867e461e..000000000 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedMeter.java +++ /dev/null @@ -1,85 +0,0 @@ -package pl.skidam.automodpack_loader_core.utils; - -import java.util.concurrent.ConcurrentSkipListMap; - -public class SpeedMeter { - - private final DownloadManager downloadManager; - private final ConcurrentSkipListMap bytesDownloadedPerSec = new ConcurrentSkipListMap<>(); - private static final int MAX_ENTRIES = 3; - - public SpeedMeter(DownloadManager downloadManager) { - this.downloadManager = downloadManager; - } - - /** - * Add new bytes to the current download tally. - */ - public synchronized void addDownloadedBytes(long newBytes) { - long bucketedTime = System.currentTimeMillis() / 1000 * 1000; - - bytesDownloadedPerSec.merge(bucketedTime, newBytes, Long::sum); - - while (bytesDownloadedPerSec.size() > MAX_ENTRIES) { - bytesDownloadedPerSec.pollFirstEntry(); - } - } - - /** - * Get the average download speed in bytes per second from the last few seconds. - */ - public long getAverageSpeedOfLastFewSeconds(int seconds) { - long totalBytes = 0; - int count = 0; - - for (Long bytes : bytesDownloadedPerSec.values()) { - totalBytes += bytes; - count++; - } - - return count >= seconds ? totalBytes / count : -1; - } - - /** - * Estimate the time remaining for the download in seconds. - */ - public long getETAInSeconds() { - long totalBytesRemaining = downloadManager.getTotalBytesRemaining(); - long speed = getAverageSpeedOfLastFewSeconds(3); - - if (speed <= 0) { - return -1; - } - - return totalBytesRemaining / speed; - } - - /** - * Format download speed in Mbps. - */ - public static String formatDownloadSpeedToMbps(long currentSpeedInBytes) { - if (currentSpeedInBytes < 0) { - return "-1"; - } - - long bitsPerSecond = currentSpeedInBytes * 8; - double mbps = bitsPerSecond / 1_000_000.0; - return String.format("%.2f Mbps", mbps); - } - - /** - * Format ETA into MM:SS format or HH:MM:SS format. - */ - public static String formatETAToSeconds(long seconds) { - seconds++; // Increment by 1 to avoid showing 00:00:00 for 0 seconds - if (seconds < 1) { - return "-1"; - } - - if (seconds < 3600) { - return String.format("%02d:%02d", seconds / 60, seconds % 60); - } - - return String.format("%02d:%02d:%02d", seconds / 3600, (seconds % 3600) / 60, seconds % 60); - } -} diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java new file mode 100644 index 000000000..41759dd7f --- /dev/null +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java @@ -0,0 +1,91 @@ +package pl.skidam.automodpack_loader_core.utils; + +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.concurrent.atomic.AtomicLong; + +public class Speedometer { + + private final AtomicLong totalBytesReceived = new AtomicLong(0); + private final AtomicLong totalBytesExpected = new AtomicLong(0); + + // Sliding Window State + private final Deque history = new ArrayDeque<>(); + + // Tuning + private static final long WINDOW_MS = 15000; // Look back few seconds for accuracy + private static final double SMOOTHING_FACTOR = 0.05; // 0.05 = Very smooth visual updates + + // Smooth Speed (For Display) + private double visualSpeed = 0; + + // Helper Class + private record Snapshot(long time, long bytes) { } + + public void addBytes(long bytes) { + totalBytesReceived.addAndGet(bytes); + update(); + } + + public void setExpectedBytes(long bytes) { + totalBytesExpected.set(bytes); + } + + private synchronized void update() { + long now = System.currentTimeMillis(); + long currentBytes = totalBytesReceived.get(); + + // 1. Add Snapshot + history.addLast(new Snapshot(now, currentBytes)); + + // 2. Prune old history + while (!history.isEmpty() && (now - history.getFirst().time > WINDOW_MS)) { + history.removeFirst(); + } + + // 3. Calculate Real-Time Window Speed + double instantSpeed = calculateWindowSpeed(now, currentBytes); + + // 4. Update Visual Speed (EMA) + if (visualSpeed == 0 || instantSpeed == 0) { + visualSpeed = instantSpeed; + } else { + visualSpeed = (instantSpeed * SMOOTHING_FACTOR) + (visualSpeed * (1.0 - SMOOTHING_FACTOR)); + } + } + + private double calculateWindowSpeed(long now, long currentBytes) { + if (history.isEmpty()) return 0.0; + + Snapshot oldest = history.getFirst(); + long timeDelta = now - oldest.time; + long bytesDelta = currentBytes - oldest.bytes; + + if (timeDelta <= 0) return 0.0; + + return ((double) bytesDelta / timeDelta) * 1000.0; + } + + public synchronized long getSpeed() { + if (history.isEmpty()) return 0; + + // Force an update if no data came in recently (handle stall) + if (System.currentTimeMillis() - history.getLast().time > 1000) { + return 0; + } + return (long) visualSpeed; + } + + public synchronized long getETA() { + long remainingBytes = Math.max(0, totalBytesExpected.get() - totalBytesReceived.get()); + if (remainingBytes == 0) return 0; + + // Get the "Real" speed from the window (not the smoothed one) + double realSpeed = calculateWindowSpeed(System.currentTimeMillis(), totalBytesReceived.get()); + + // Safety: If speed is 0 or very low, return -1 (unknown) + if (realSpeed < 1024) return -1; + + return (long) (remainingBytes / realSpeed); + } +} \ No newline at end of file diff --git a/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java b/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java index aac46d342..d10286042 100644 --- a/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java +++ b/src/main/java/pl/skidam/automodpack/client/ui/DownloadScreen.java @@ -4,7 +4,6 @@ import static pl.skidam.automodpack_core.Constants.clientConfigFile; import net.minecraft.ChatFormatting; -import net.minecraft.util.Util; import net.minecraft.client.gui.components.Button; import net.minecraft.network.chat.Component; import net.minecraft.network.chat.MutableComponent; @@ -17,27 +16,32 @@ import pl.skidam.automodpack_core.config.ConfigTools; import pl.skidam.automodpack_loader_core.screen.ScreenManager; import pl.skidam.automodpack_loader_core.utils.DownloadManager; -import pl.skidam.automodpack_loader_core.utils.SpeedMeter; +import pl.skidam.automodpack_loader_core.utils.SpeedFormatter; public class DownloadScreen extends VersionedScreen { - // thank you mojang for textures, i am sorry that i have to bundle them myself but i dont want to deal with atlas textures on multiversion setup private static final Identifier PROGRESS_BAR_EMPTY_TEXTURE = Common.id("textures/gui/sprites/green_background.png"); private static final Identifier PROGRESS_BAR_FULL_TEXTURE = Common.id("textures/gui/sprites/green_progress.png"); private static final int PROGRESS_BAR_WIDTH = 182; private static final int PROGRESS_BAR_HEIGHT = 5; + private final DownloadManager downloadManager; private final String header; + private long ticks = 0; private boolean musicStarted = false; private Button cancelButton; private Button muteMusicButton; private Button playMusicButton; - private String lastStage = "-1"; - private int lastPercentage = -1; - private String lastSpeed = "-1"; - private String lastETA = "-1"; + // UI Cache + private String cachedStage = "0/0"; + private double cachedPercentage = 0.0; + private String cachedSpeed = "0 B/s"; + private String cachedETA = "Calculating..."; + + private long lastTextUpdate = 0; + private static final long TEXT_UPDATE_INTERVAL = 100; // Update strings 10x per second public DownloadScreen(DownloadManager downloadManager, String header) { super(VersionedText.literal("DownloadScreen")); @@ -48,103 +52,79 @@ public DownloadScreen(DownloadManager downloadManager, String header) { @Override protected void init() { super.init(); - initWidgets(); - - Util.backgroundExecutor().execute(() -> { - while (downloadManager != null && downloadManager.isRunning()) { - lastStage = downloadManager.getStage(); - lastPercentage = downloadManager.getTotalPercentageOfFileSizeDownloaded(); - lastSpeed = SpeedMeter.formatDownloadSpeedToMbps(downloadManager.getSpeedMeter().getAverageSpeedOfLastFewSeconds(1)); - lastETA = SpeedMeter.formatETAToSeconds(downloadManager.getSpeedMeter().getETAInSeconds()); - } - }); } private void initWidgets() { cancelButton = addRenderableWidget( - buttonWidget(this.width / 2 - 60, this.height / 2 + 80, 120, 20, - VersionedText.translatable("automodpack.cancel"), - button -> { - cancelButton.active = false; - cancelDownload(); - AudioManager.stopMusic(); - } - ) + buttonWidget(this.width / 2 - 60, this.height / 2 + 80, 120, 20, + VersionedText.translatable("automodpack.cancel"), + button -> { + cancelButton.active = false; + cancelDownload(); + AudioManager.stopMusic(); + } + ) ); int x = this.width - 40; int y = this.height - 40; muteMusicButton = addRenderableWidget( - VersionedScreen.iconButtonWidget(x, y, 20, 8, - button -> { - AudioManager.stopMusic(); - clientConfig.playMusic = false; - ConfigTools.save(clientConfigFile, clientConfig); - }, - "music-note" - ) + VersionedScreen.iconButtonWidget(x, y, 20, 8, + button -> { + AudioManager.stopMusic(); + clientConfig.playMusic = false; + ConfigTools.save(clientConfigFile, clientConfig); + }, "music-note" + ) ); playMusicButton = addRenderableWidget( - VersionedScreen.iconButtonWidget( - x, - y, - 20, - 8, - button -> { - AudioManager.playMusic(); - clientConfig.playMusic = true; - ConfigTools.save(clientConfigFile, clientConfig); - }, - "mute-music-note" - ) + VersionedScreen.iconButtonWidget(x, y, 20, 8, + button -> { + AudioManager.playMusic(); + clientConfig.playMusic = true; + ConfigTools.save(clientConfigFile, clientConfig); + }, "mute-music-note" + ) ); } - private Component getStage() { - if (lastStage.equals("-1")) { - return VersionedText.translatable( - "automodpack.download.calculating" - ); - } - return VersionedText.literal(lastStage); - } + private void updateUIState() { + if (downloadManager == null || !downloadManager.isRunning()) return; + + long now = System.currentTimeMillis(); + if (now - lastTextUpdate >= TEXT_UPDATE_INTERVAL) { + lastTextUpdate = now; - private Component getPercentage() { - if (lastPercentage == -1) { - return VersionedText.translatable( - "automodpack.download.calculating" - ); + cachedStage = downloadManager.getStage(); + cachedPercentage = downloadManager.getPrecisePercentage(); + cachedSpeed = SpeedFormatter.formatSpeed(downloadManager.getDownloadSpeed()); + cachedETA = SpeedFormatter.formatETA(downloadManager.getETA()); } - return VersionedText.literal(lastPercentage + "%"); } + // --- Components --- + + private Component getStage() { return VersionedText.literal(cachedStage); } + private Component getPercentage() { return VersionedText.literal((int) cachedPercentage + "%"); } private Component getTotalDownloadSpeed() { - if (lastSpeed.equals("-1")) { - return VersionedText.translatable( - "automodpack.download.calculating" - ); - } - return VersionedText.literal(lastSpeed); + return "-1".equals(cachedSpeed) + ? VersionedText.translatable("automodpack.download.calculating") + : VersionedText.literal(cachedSpeed); } - private Component getTotalETA() { - if (lastETA.equals("-1")) { - return VersionedText.translatable( - "automodpack.download.calculating" - ); - } - return VersionedText.translatable("automodpack.download.eta", lastETA); // Time left: %s + return "-1".equals(cachedETA) + ? VersionedText.translatable("automodpack.download.calculating") + : VersionedText.translatable("automodpack.download.eta", cachedETA); } private float getDownloadScale() { - return Math.max(0, Math.min(100, lastPercentage)) * 0.01F; // Convert the clamped percentage to a scale between 0.0f and 1.0f + return (float) (Math.max(0.0, Math.min(100.0, cachedPercentage)) * 0.01); } private void drawDownloadingFiles(VersionedMatrices matrices) { - int lineHeight = 12; // Consistent line spacing float scale = 1.0F; int y = this.height / 2 - 90; @@ -152,183 +132,103 @@ private void drawDownloadingFiles(VersionedMatrices matrices) { matrices.scale(scale, scale, scale); if (downloadManager != null && !downloadManager.downloadsInProgress.isEmpty()) { - drawCenteredTextWithShadow( - matrices, - this.font, - VersionedText.translatable( - "automodpack.download.downloading" - ).withStyle(ChatFormatting.BOLD), - this.width / 2, - y, - TextColors.WHITE - ); - - // Use a separate variable for the current y position + drawCenteredTextWithShadow(matrices, this.font, + VersionedText.translatable("automodpack.download.downloading").withStyle(ChatFormatting.BOLD), + this.width / 2, y, TextColors.WHITE); + int currentY = y + 15; synchronized (downloadManager.downloadsInProgress) { - for (DownloadManager.DownloadData downloadData : downloadManager.downloadsInProgress.values()) { - String text = downloadData.getFileName(); - drawCenteredTextWithShadow( - matrices, - this.font, - VersionedText.literal(text), - (int) (((float) this.width / 2) * scale), - currentY, - TextColors.GRAY - ); + for (DownloadManager.DownloadData data : downloadManager.downloadsInProgress.values()) { + drawCenteredTextWithShadow(matrices, this.font, + VersionedText.literal(data.getFileName()), + (int) (((float) this.width / 2) * scale), currentY, TextColors.GRAY); currentY += 10; } } } else { - drawCenteredTextWithShadow( - matrices, - this.font, - VersionedText.translatable("automodpack.download.noFiles"), - (int) (((float) this.width / 2) * scale), - y, - TextColors.WHITE - ); - drawCenteredTextWithShadow( - matrices, - this.font, - VersionedText.translatable("automodpack.wait").withStyle( - ChatFormatting.BOLD - ), - (int) (((float) this.width / 2) * scale), - y + lineHeight * 2, - TextColors.WHITE - ); + drawCenteredTextWithShadow(matrices, this.font, + VersionedText.translatable("automodpack.download.noFiles"), + (int) (((float) this.width / 2) * scale), y, TextColors.WHITE); + drawCenteredTextWithShadow(matrices, this.font, + VersionedText.translatable("automodpack.wait").withStyle(ChatFormatting.BOLD), + (int) (((float) this.width / 2) * scale), y + 24, TextColors.WHITE); } - matrices.popPose(); } - private void checkAndStartMusic() { - if (ticks++ <= 30) { - muteMusicButton.active = false; - playMusicButton.active = false; - return; - } - - muteMusicButton.active = true; - playMusicButton.active = true; - - if (musicStarted) { - return; - } - - if (clientConfig.playMusic) { - AudioManager.playMusic(); - } - - musicStarted = true; - } - @Override public void versionedRender(VersionedMatrices matrices, int mouseX, int mouseY, float delta) { - int lineHeight = 12; // Consistent line spacing + updateUIState(); + int lineHeight = 12; drawDownloadingFiles(matrices); // Title - MutableComponent titleText = VersionedText.literal(header).withStyle( - ChatFormatting.BOLD - ); - drawCenteredTextWithShadow( - matrices, - this.font, - titleText, - this.width / 2, - this.height / 2 - 110, - TextColors.WHITE - ); + drawCenteredTextWithShadow(matrices, this.font, + VersionedText.literal(header).withStyle(ChatFormatting.BOLD), + this.width / 2, this.height / 2 - 110, TextColors.WHITE); if (downloadManager != null && downloadManager.isRunning()) { -// MutableComponent percentage = (MutableComponent) this.getPercentage(); - MutableComponent stage = (MutableComponent) this.getStage(); - MutableComponent eta = (MutableComponent) this.getTotalETA(); - MutableComponent speed = (MutableComponent) this.getTotalDownloadSpeed(); - - // Stage - drawCenteredTextWithShadow( - matrices, - this.font, - stage, - this.width / 2, - this.height / 2 - 10, - TextColors.WHITE - ); - - // ETA - drawCenteredTextWithShadow( - matrices, - this.font, - eta, - this.width / 2, - this.height / 2 - 10 + lineHeight * 2, - TextColors.WHITE - ); - - // scale it to make it ~250px instead of 182px - float scaleBar = 1.35F; + drawCenteredTextWithShadow(matrices, this.font, (MutableComponent) getStage(), this.width / 2, this.height / 2 - 10, TextColors.WHITE); + drawCenteredTextWithShadow(matrices, this.font, (MutableComponent) getTotalETA(), this.width / 2, this.height / 2 - 10 + lineHeight * 2, TextColors.WHITE); + float scaleBar = 1.35F; int barWidth = PROGRESS_BAR_WIDTH; int barHeight = PROGRESS_BAR_HEIGHT; int barFilledWidth = (int) (barWidth * getDownloadScale()); int barYPos = this.height / 2 + 36; + float barDrawX = (this.width - barWidth * scaleBar) / 2.0F / scaleBar; float barDrawY = barYPos / scaleBar; - int barScreenX = Math.round(barDrawX); - int barScreenY = Math.round(barDrawY); matrices.pushPose(); matrices.scale(scaleBar, scaleBar, scaleBar); - - drawTexture(PROGRESS_BAR_EMPTY_TEXTURE, matrices, barScreenX, barScreenY, 0, 0, barWidth, barHeight, barWidth, barHeight); - drawTexture(PROGRESS_BAR_FULL_TEXTURE, matrices, barScreenX, barScreenY, 0, 0, Math.min(barFilledWidth, barWidth), barHeight, barWidth, barHeight); - + drawTexture(PROGRESS_BAR_EMPTY_TEXTURE, matrices, Math.round(barDrawX), Math.round(barDrawY), 0, 0, barWidth, barHeight, barWidth, barHeight); + drawTexture(PROGRESS_BAR_FULL_TEXTURE, matrices, Math.round(barDrawX), Math.round(barDrawY), 0, 0, Math.min(barFilledWidth, barWidth), barHeight, barWidth, barHeight); matrices.popPose(); - // Speed - drawCenteredTextWithShadow( - matrices, - this.font, - speed, - this.width / 2, - this.height / 2 + 36 + lineHeight * 2, - TextColors.WHITE - ); - + drawCenteredTextWithShadow(matrices, this.font, (MutableComponent) getTotalDownloadSpeed(), this.width / 2, this.height / 2 + 36 + lineHeight * 2, TextColors.WHITE); cancelButton.active = true; } else { cancelButton.active = false; } checkAndStartMusic(); + updateMusicButtons(); + } + + private void updateMusicButtons() { if (playMusicButton.active && muteMusicButton.active) { - boolean musicPlaying = AudioManager.isMusicPlaying(); - muteMusicButton.visible = musicPlaying; - playMusicButton.visible = !musicPlaying; + boolean playing = AudioManager.isMusicPlaying(); + muteMusicButton.visible = playing; + playMusicButton.visible = !playing; } else { muteMusicButton.visible = clientConfig.playMusic; playMusicButton.visible = !clientConfig.playMusic; } } - @Override - public boolean shouldCloseOnEsc() { - return false; + private void checkAndStartMusic() { + if (ticks++ <= 30) { + muteMusicButton.active = false; + playMusicButton.active = false; + return; + } + muteMusicButton.active = true; + playMusicButton.active = true; + + if (musicStarted) return; + if (clientConfig.playMusic) AudioManager.playMusic(); + musicStarted = true; } + @Override + public boolean shouldCloseOnEsc() { return false; } + public void cancelDownload() { try { - if (downloadManager != null) { - downloadManager.cancelAllAndShutdown(); - } - + if (downloadManager != null) downloadManager.cancelAllAndShutdown(); new ScreenManager().title(); - } catch (Exception e) { - e.printStackTrace(); - } + } catch (Exception e) { e.printStackTrace(); } } -} +} \ No newline at end of file From 13f5f1e36b382bbd66a7197a1d554e6835f0774b Mon Sep 17 00:00:00 2001 From: skidam Date: Mon, 9 Feb 2026 17:35:32 +0100 Subject: [PATCH 20/22] Implement clever file download scheduler to avoid "The Tail of Death" situation which was making speedometer very inconsistant. E.g. AutoModpack was downloading 5 large files at high speed, while 4,000 small files sit in the queue. The progress bar hit 90% quickly (bytes downloaded), but then hangs for minutes while automodpack processed the remaining 4,000 small files one by one. Giving "The downloader is stuck/broken." perception. Now we are very picky which file we choose to download preveting situations that we only download from a single source at one time or first download only big files from source A and we left with a massive log of little files for clients with high latency to the server each request requires a roundtrip to the server which slows down the download a lot, implementing batching request will improve that yet more. Gemini 3 Pro helped there with this commit. --- .../automodpack_loader_core/SelfUpdater.java | 1 + .../client/ModpackUpdater.java | 30 +- .../utils/DownloadManager.java | 279 ++++++++++++------ 3 files changed, 207 insertions(+), 103 deletions(-) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java index feab6a00a..62077c532 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/SelfUpdater.java @@ -175,6 +175,7 @@ public static void installModVersion(ModrinthAPI automodpack) { automodpackUpdateJar, automodpack.SHA1Hash(), List.of(automodpack.downloadUrl()), + automodpack.fileSize(), () -> LOGGER.info("Downloaded update for AutoModpack."), () -> LOGGER.error("Failed to download update for AutoModpack.") ); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java index d9b93114a..6607af5a1 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/client/ModpackUpdater.java @@ -275,12 +275,11 @@ private void downloadModpack(Set new ScreenManager().download(downloadManager, getModpackName()); downloadManager.attachDownloadClient(downloadClient); - var randomizedList = new ArrayList<>(finalFilesToUpdate); - Collections.shuffle(randomizedList); - for (var serverItem : randomizedList) { + for (var serverItem : finalFilesToUpdate) { String serverFilePath = serverItem.file; - String serverHash = serverItem.sha1; + String serverFileHash = serverItem.sha1; + long serverFileSize = Long.parseLong(serverItem.size); Path downloadFile = SmartFileUtils.getPath(modpackDir, serverFilePath); @@ -289,8 +288,8 @@ private void downloadModpack(Set } List urls = new ArrayList<>(); - if (fetchManager != null && fetchManager.getFetchDatas().containsKey(serverHash)) { - urls.addAll(fetchManager.getFetchDatas().get(serverHash).fetchedData().urls()); + if (fetchManager != null && fetchManager.getFetchDatas().containsKey(serverFileHash)) { + urls.addAll(fetchManager.getFetchDatas().get(serverFileHash).fetchedData().urls()); } Runnable failureCallback = () -> { @@ -299,21 +298,21 @@ private void downloadModpack(Set Runnable successCallback = () -> { List mainPageUrls = new LinkedList<>(); - if (fetchManager != null && fetchManager.getFetchDatas().get(serverHash) != null) { - mainPageUrls = fetchManager.getFetchDatas().get(serverHash).fetchedData().mainPageUrls(); + if (fetchManager != null && fetchManager.getFetchDatas().get(serverFileHash) != null) { + mainPageUrls = fetchManager.getFetchDatas().get(serverFileHash).fetchedData().mainPageUrls(); } changelogs.changesAddedList.put(downloadFile.getFileName().toString(), mainPageUrls); try { - cache.overwriteCache(downloadFile, serverHash); + cache.overwriteCache(downloadFile, serverFileHash); } catch (Exception e) { LOGGER.error("Failed to update cache for {}", downloadFile, e); } }; - downloadManager.download(downloadFile, serverHash, urls, successCallback, failureCallback); + downloadManager.download(downloadFile, serverFileHash, urls, serverFileSize, successCallback, failureCallback); } downloadManager.joinAll(); @@ -385,12 +384,11 @@ private void downloadModpack(Set // TODO try to fetch again from modrinth and curseforge - randomizedList = new ArrayList<>(refreshedFilteredList); - Collections.shuffle(randomizedList); - for (var serverItem : randomizedList) { + for (var serverItem : refreshedFilteredList) { String serverFilePath = serverItem.file; - String serverHash = serverItem.sha1; + String serverFileHash = serverItem.sha1; + long serverFileSize = Long.parseLong(serverItem.size); Path downloadFile = SmartFileUtils.getPath(modpackDir, serverFilePath); @@ -404,13 +402,13 @@ private void downloadModpack(Set changelogs.changesAddedList.put(downloadFile.getFileName().toString(), null); try { - cache.overwriteCache(downloadFile, serverHash); + cache.overwriteCache(downloadFile, serverFileHash); } catch (Exception e) { LOGGER.error("Failed to update cache for {}", downloadFile, e); } }; - downloadManager.download(downloadFile, serverHash, List.of(), successCallback, failureCallback); + downloadManager.download(downloadFile, serverFileHash, List.of(), serverFileSize, successCallback, failureCallback); } downloadManager.joinAll(); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java index 31b3e8f17..b36b57ce1 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java @@ -35,14 +35,16 @@ public class DownloadManager { private final Map queuedDownloads = new ConcurrentHashMap<>(); public final Map downloadsInProgress = new ConcurrentHashMap<>(); - // Stats - private final AtomicLong totalBytesDownloaded = new AtomicLong(0); // For Progress Bar (Includes Cache) - private final AtomicLong totalBytesToDownload = new AtomicLong(0); // For Progress Bar - private int addedToQueue = 0; + // --- Source Usage Tracking --- + private final Map activeDownloadsPerSource = new ConcurrentHashMap<>(); + + // --- PROGRESS TRACKING --- + private final AtomicLong totalBytesToDownload = new AtomicLong(0); + private final AtomicLong totalBytesDownloaded = new AtomicLong(0); + private int totalFilesAdded = 0; private int downloadedCount = 0; private final Semaphore semaphore = new Semaphore(0); - private final Speedometer speedometer = new Speedometer(); public DownloadManager() { } @@ -56,14 +58,13 @@ public void attachDownloadClient(DownloadClient downloadClient) { this.downloadClient = downloadClient; } - public void download(Path file, String sha1, List urls, Runnable successCallback, Runnable failureCallback) { + public synchronized void download(Path file, String sha1, List urls, long fileSize, Runnable successCallback, Runnable failureCallback) { FileInspection.HashPathPair hashPathPair = new FileInspection.HashPathPair(sha1, file); - if (queuedDownloads.containsKey(hashPathPair)) return; - queuedDownloads.put(hashPathPair, new QueuedDownload(file, urls, 0, successCallback, failureCallback)); - addedToQueue++; - + QueuedDownload task = new QueuedDownload(file, urls, fileSize, 0, successCallback, failureCallback); + queuedDownloads.put(hashPathPair, task); + totalFilesAdded++; downloadNext(); } @@ -72,11 +73,126 @@ private synchronized void downloadNext() { return; } - var entry = queuedDownloads.entrySet().iterator().next(); - FileInspection.HashPathPair key = entry.getKey(); - QueuedDownload task = queuedDownloads.remove(key); + // --- 1. CALCULATE METRICS --- + + long totalBytes = totalBytesToDownload.get(); + if (totalBytes <= 0) totalBytes = 1; + if (totalFilesAdded <= 0) totalFilesAdded = 1; + + // Dynamic Average (Pivot for Big vs Small) + long avgSize = totalBytes / totalFilesAdded; + + // Calculate Progress Percentages (0.00 to 1.00) + double byteProgress = (double) totalBytesDownloaded.get() / totalBytes; + double fileProgress = (double) downloadedCount / totalFilesAdded; + + // Calculate LAG + // Example: 50% Bytes Done, 40% Files Done -> Lag = 0.10 (BAD) + double lag = byteProgress - fileProgress; + + // --- 2. DETERMINE SLOT ALLOCATION --- + + int maxBigSlots; + + if (lag > 0.02) { + // Files are >2% behind. + // Don't queue any big files anymore. Use all threads for Small Files. + maxBigSlots = 0; + } else if (lag > 0.005) { + // Files are >0.5% behind. + // Allow only 1 big file to keep bandwidth alive. + maxBigSlots = 1; + } else { + // Balanced / Ahead. + // Allow natural mix 3:2. + maxBigSlots = 2; + } + + // --- 3. COUNT CURRENT STATE --- + + int activeBig = 0; + int activeSmall = 0; + for (DownloadData d : downloadsInProgress.values()) { + if (d.fileSize > avgSize) activeBig++; + else activeSmall++; + } + + // --- 4. DECISION --- + + // Do we want a Big file? + boolean wantBig = (activeBig < maxBigSlots); + + // --- 5. AVAILABILITY CHECK --- + + boolean hasBig = false; + boolean hasSmall = false; + + // Fast scan + for (QueuedDownload t : queuedDownloads.values()) { + if (t.fileSize > avgSize) hasBig = true; + else hasSmall = true; + if (hasBig && hasSmall) break; // Found both + } + + // Fallback Logic + if (wantBig && !hasBig) wantBig = false; // Wanted Big, but none left. Take Small. + if (!wantBig && !hasSmall) wantBig = true; // Wanted Small, but none left. Take Big. + + // --- 6. SELECT BEST FILE --- + + FileInspection.HashPathPair bestKey = null; + QueuedDownload bestTask = null; + String bestDomain = null; + int lowestLoad = Integer.MAX_VALUE; + + for (Map.Entry entry : queuedDownloads.entrySet()) { + QueuedDownload task = entry.getValue(); + boolean isBig = task.fileSize > avgSize; + + // FILTER: Strict Type Check + if (isBig != wantBig) continue; + + String source = predictSource(task); + int activeInSource = activeDownloadsPerSource.getOrDefault(source, 0); + + // Source Cap (Optional: set to 2 or 3 per source if needed) + if (activeInSource >= MAX_DOWNLOADS_IN_PROGRESS) continue; + + // Load Balancing: Pick least busy source + if (activeInSource < lowestLoad) { + lowestLoad = activeInSource; + bestKey = entry.getKey(); + bestTask = task; + bestDomain = source; + } + } + + // FINAL FALLBACK: + // If strict filtering failed (e.g. we wanted Small but all Small domains are capped), + // we MUST pick something else to avoid idling threads. + if (bestTask == null) { + // Try to find *any* valid download regardless of size + for (Map.Entry entry : queuedDownloads.entrySet()) { + QueuedDownload task = entry.getValue(); + String source = predictSource(task); + if (activeDownloadsPerSource.getOrDefault(source, 0) < MAX_DOWNLOADS_IN_PROGRESS) { + bestKey = entry.getKey(); + bestTask = task; + bestDomain = source; + break; + } + } + } + + if (bestTask == null) return; + + // --- EXECUTE --- + queuedDownloads.remove(bestKey); + activeDownloadsPerSource.merge(bestDomain, 1, Integer::sum); - if (task == null) return; + final FileInspection.HashPathPair key = bestKey; + final QueuedDownload task = bestTask; + final String activeDomain = bestDomain; CompletableFuture future = CompletableFuture.runAsync(() -> { try { @@ -86,12 +202,34 @@ private synchronized void downloadNext() { } }, downloadExecutor); - downloadsInProgress.put(key, new DownloadData(future, task.file)); + downloadsInProgress.put(key, new DownloadData(future, task.file, activeDomain, task.fileSize)); } - private void processDownloadTask(FileInspection.HashPathPair hashPathPair, QueuedDownload task) { - LOGGER.info("Processing {} - Hash: {}", task.file.getFileName(), hashPathPair.hash()); + private String predictSource(QueuedDownload task) { + int numberOfIndexes = task.urls.size(); + int urlIndex = Math.min(task.attempts / MAX_DOWNLOAD_ATTEMPTS, numberOfIndexes); + if (task.urls.size() > urlIndex) { + String url = task.urls.get(urlIndex); + if (!Objects.equals(url, "host")) { + return getDomainFromUrl(url); + } + } + return "internal_client"; + } + private String getDomainFromUrl(String url) { + if (url == null) return "unknown"; + try { + int protocolEnd = url.indexOf("://"); + String noProtocol = (protocolEnd > -1) ? url.substring(protocolEnd + 3) : url; + int slash = noProtocol.indexOf('/'); + return (slash > -1) ? noProtocol.substring(0, slash) : noProtocol; + } catch (Exception e) { + return "unknown"; + } + } + + private void processDownloadTask(FileInspection.HashPathPair hashPathPair, QueuedDownload task) { Path storeFile = storeDir.resolve(hashPathPair.hash()); boolean success = false; boolean interrupted = false; @@ -109,7 +247,6 @@ private void processDownloadTask(FileInspection.HashPathPair hashPathPair, Queue // DOWNLOAD REQUIRED success = attemptDownload(hashPathPair, task, storeFile); } - } catch (InterruptedException e) { interrupted = true; } catch (Exception e) { @@ -123,7 +260,6 @@ private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, Queued int numberOfIndexes = task.urls.size(); int urlIndex = Math.min(task.attempts / MAX_DOWNLOAD_ATTEMPTS, numberOfIndexes); String url = (task.urls.size() > urlIndex) ? task.urls.get(urlIndex) : null; - Path tempStoreFile = storeDir.resolve(hashPathPair.hash() + ".tmp"); try { @@ -132,7 +268,6 @@ private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, Queued } else if (downloadClient != null) { hostDownloadFile(hashPathPair, tempStoreFile); } else { - LOGGER.error("No valid source found for {}", task.file.getFileName()); return false; } @@ -145,14 +280,19 @@ private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, Queued return false; } } catch (IOException e) { - LOGGER.warn("Download I/O error for {}: {}", task.file.getFileName(), e.getMessage()); SmartFileUtils.executeOrder66(tempStoreFile); return false; } } private void cleanupAndFinalize(FileInspection.HashPathPair key, QueuedDownload task, Path storeFile, boolean success, boolean interrupted) { - downloadsInProgress.remove(key); + DownloadData data = downloadsInProgress.remove(key); + + if (data != null && data.activeDomain != null) { + synchronized (this) { + activeDownloadsPerSource.compute(data.activeDomain, (k, v) -> (v == null || v <= 1) ? null : v - 1); + } + } if (success) { try { @@ -176,22 +316,15 @@ private void cleanupAndFinalize(FileInspection.HashPathPair key, QueuedDownload private void handleRetry(FileInspection.HashPathPair key, QueuedDownload task, boolean interrupted) { if (interrupted) return; - try { if (Files.exists(task.file)) { - long size = Files.size(task.file); - // If we retry, we expect to download this size again - totalBytesToDownload.addAndGet(size); + totalBytesToDownload.addAndGet(Files.size(task.file)); speedometer.setExpectedBytes(totalBytesToDownload.get()); } } catch (IOException ignored) {} - SmartFileUtils.executeOrder66(task.file); - int numberOfIndexes = task.urls.size(); - int maxAttempts = (numberOfIndexes + 1) * MAX_DOWNLOAD_ATTEMPTS; - - if (task.attempts < maxAttempts) { + if (task.attempts < (task.urls.size() + 1) * MAX_DOWNLOAD_ATTEMPTS) { LOGGER.warn("Retrying download: {}", task.file.getFileName()); task.attempts++; queuedDownloads.put(key, task); @@ -202,7 +335,7 @@ private void handleRetry(FileInspection.HashPathPair key, QueuedDownload task, b } } - // --- Download Implementations --- + // --- IO --- private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, Path targetFile) throws IOException { SmartFileUtils.createParentDirs(targetFile); @@ -212,8 +345,6 @@ private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, Path tar private void httpDownloadFile(String urlString, Path targetFile) throws IOException, InterruptedException { SmartFileUtils.createParentDirs(targetFile); - LOGGER.info("Downloading from {}", urlString); - URLConnection connection = getHttpConnection(urlString); try (InputStream rawIn = connection.getInputStream(); @@ -240,9 +371,6 @@ private URLConnection getHttpConnection(String urlString) throws IOException { return connection; } - /** - * Updates counters ONLY for actual network traffic. - */ private void updateNetworkProgress(long bytes) { totalBytesDownloaded.addAndGet(bytes); speedometer.addBytes(bytes); @@ -250,30 +378,19 @@ private void updateNetworkProgress(long bytes) { private boolean verifyFile(Path file, String expectedHash) { if (!Files.exists(file)) return false; - try { - return Objects.equals(HashUtils.getHash(file), expectedHash); - } catch (Exception e) { - return false; - } + try { return Objects.equals(HashUtils.getHash(file), expectedHash); } catch (Exception e) { return false; } } - // --- Getters & Control --- - public void joinAll() throws InterruptedException { - semaphore.acquire(addedToQueue); + semaphore.acquire(totalFilesAdded); if (downloadExecutor.isShutdown()) throw new InterruptedException(); - semaphore.release(addedToQueue); + semaphore.release(totalFilesAdded); } // --- UI Helpers --- - public long getDownloadSpeed() { - return speedometer.getSpeed(); - } - - public long getETA() { - return speedometer.getETA(); - } + public long getDownloadSpeed() { return speedometer.getSpeed(); } + public long getETA() { return speedometer.getETA(); } public double getPrecisePercentage() { long total = totalBytesToDownload.get(); @@ -282,62 +399,50 @@ public double getPrecisePercentage() { return Math.max(0.0, Math.min(100.0, pc)); } - public String getStage() { return downloadedCount + "/" + addedToQueue; } - + public String getStage() { return downloadedCount + "/" + totalFilesAdded; } public boolean isRunning() { return !downloadExecutor.isShutdown(); } public void cancelAllAndShutdown() { cancelled = true; queuedDownloads.clear(); - downloadsInProgress.forEach((key, data) -> { - data.future.cancel(true); - SmartFileUtils.executeOrder66(data.file); + downloadsInProgress.forEach((k, v) -> { + v.future.cancel(true); + SmartFileUtils.executeOrder66(v.file); }); - - semaphore.release(addedToQueue); + semaphore.release(totalFilesAdded); downloadsInProgress.clear(); downloadedCount = 0; - addedToQueue = 0; - if (downloadClient != null) downloadClient.close(); downloadExecutor.shutdown(); } - public boolean isCancelled() { - return cancelled; - } + public boolean isCancelled() { return cancelled; } + public void setCancelled(boolean cancelled) { this.cancelled = cancelled; } - public void setCancelled(boolean cancelled) { - this.cancelled = cancelled; - } + // --- Inner Classes --- public static class QueuedDownload { - private final Path file; - private final List urls; - private int attempts; - private final Runnable successCallback; - private final Runnable failureCallback; - - public QueuedDownload(Path file, List urls, int attempts, Runnable successCallback, Runnable failureCallback) { - this.file = file; - this.urls = urls; - this.attempts = attempts; - this.successCallback = successCallback; - this.failureCallback = failureCallback; + public final Path file; + public final List urls; + public final long fileSize; + public int attempts; + public final Runnable successCallback; + public final Runnable failureCallback; + + public QueuedDownload(Path f, List u, long size, int a, Runnable s, Runnable fa) { + file = f; urls = u; fileSize = size; attempts = a; successCallback = s; failureCallback = fa; } } public static class DownloadData { public CompletableFuture future; public Path file; + public String activeDomain; + public long fileSize; - DownloadData(CompletableFuture future, Path file) { - this.future = future; - this.file = file; - } - - public String getFileName() { - return file.getFileName().toString(); + DownloadData(CompletableFuture f, Path p, String d, long s) { + future = f; file = p; activeDomain = d; fileSize = s; } + public String getFileName() { return file.getFileName().toString(); } } } \ No newline at end of file From 318994cac5bee79e20f1a3cd31e6fdc80e0d4930 Mon Sep 17 00:00:00 2001 From: skidam Date: Mon, 9 Feb 2026 19:47:58 +0100 Subject: [PATCH 21/22] Reuse single HTTP client among other smaller improvements --- .../utils/DownloadManager.java | 93 ++++++------------- .../utils/HttpFileDownloader.java | 90 ++++++++++++++++++ 2 files changed, 120 insertions(+), 63 deletions(-) create mode 100644 loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/HttpFileDownloader.java diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java index b36b57ce1..e63c74c4e 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/DownloadManager.java @@ -1,6 +1,5 @@ package pl.skidam.automodpack_loader_core.utils; -import pl.skidam.automodpack_core.protocol.NetUtils; import pl.skidam.automodpack_core.utils.SmartFileUtils; import pl.skidam.automodpack_core.utils.HashUtils; import pl.skidam.automodpack_core.utils.CustomThreadFactoryBuilder; @@ -8,14 +7,13 @@ import pl.skidam.automodpack_core.protocol.DownloadClient; import java.io.*; -import java.net.*; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicLong; -import java.util.zip.GZIPInputStream; +import java.util.function.IntConsumer; import static pl.skidam.automodpack_core.Constants.*; @@ -29,13 +27,15 @@ public class DownloadManager { new CustomThreadFactoryBuilder().setNameFormat("AutoModpackDownload-%d").build() ); + private final HttpFileDownloader httpDownloader = new HttpFileDownloader(); private DownloadClient downloadClient = null; + private volatile boolean cancelled = false; + // --- QUEUES --- private final Map queuedDownloads = new ConcurrentHashMap<>(); public final Map downloadsInProgress = new ConcurrentHashMap<>(); - // --- Source Usage Tracking --- private final Map activeDownloadsPerSource = new ConcurrentHashMap<>(); // --- PROGRESS TRACKING --- @@ -90,23 +90,19 @@ private synchronized void downloadNext() { // Example: 50% Bytes Done, 40% Files Done -> Lag = 0.10 (BAD) double lag = byteProgress - fileProgress; - // --- 2. DETERMINE SLOT ALLOCATION --- + // --- 2. DETERMINE SHARES (Proportional Control) --- + // We decide what % of our threads should be working on Big Files. + double targetBigShare; - int maxBigSlots; + if (lag > 0.02) targetBigShare = 0.0; // Panic (>2% Behind): 0% Big, 100% Small + else if (lag > 0.005) targetBigShare = 0.2; // Warning (>0.5% Behind): 20% Big (1/5) + else if (lag < -0.15) targetBigShare = 1.0; // Ahead (>15%): 100% Big + else if (lag < -0.10) targetBigShare = 0.8; // Ahead (>10%): 80% Big (4/5) + else if (lag < -0.05) targetBigShare = 0.6; // Ahead (>5%): 60% Big (3/5) + else targetBigShare = 0.4; // Balanced: 40% Big (2/5) - if (lag > 0.02) { - // Files are >2% behind. - // Don't queue any big files anymore. Use all threads for Small Files. - maxBigSlots = 0; - } else if (lag > 0.005) { - // Files are >0.5% behind. - // Allow only 1 big file to keep bandwidth alive. - maxBigSlots = 1; - } else { - // Balanced / Ahead. - // Allow natural mix 3:2. - maxBigSlots = 2; - } + int slotsForBig = (int) Math.round(MAX_DOWNLOADS_IN_PROGRESS * targetBigShare); + int slotsForSmall = MAX_DOWNLOADS_IN_PROGRESS - slotsForBig; // --- 3. COUNT CURRENT STATE --- @@ -119,8 +115,7 @@ private synchronized void downloadNext() { // --- 4. DECISION --- - // Do we want a Big file? - boolean wantBig = (activeBig < maxBigSlots); + boolean preferBig = activeBig < slotsForBig || activeSmall > slotsForSmall; // --- 5. AVAILABILITY CHECK --- @@ -135,14 +130,14 @@ private synchronized void downloadNext() { } // Fallback Logic - if (wantBig && !hasBig) wantBig = false; // Wanted Big, but none left. Take Small. - if (!wantBig && !hasSmall) wantBig = true; // Wanted Small, but none left. Take Big. + if (preferBig && !hasBig) preferBig = false; // Wanted Big, but none left. Take Small. + if (!preferBig && !hasSmall) preferBig = true; // Wanted Small, but none left. Take Big. // --- 6. SELECT BEST FILE --- FileInspection.HashPathPair bestKey = null; QueuedDownload bestTask = null; - String bestDomain = null; + String bestSource = null; int lowestLoad = Integer.MAX_VALUE; for (Map.Entry entry : queuedDownloads.entrySet()) { @@ -150,7 +145,7 @@ private synchronized void downloadNext() { boolean isBig = task.fileSize > avgSize; // FILTER: Strict Type Check - if (isBig != wantBig) continue; + if (isBig != preferBig) continue; String source = predictSource(task); int activeInSource = activeDownloadsPerSource.getOrDefault(source, 0); @@ -163,7 +158,7 @@ private synchronized void downloadNext() { lowestLoad = activeInSource; bestKey = entry.getKey(); bestTask = task; - bestDomain = source; + bestSource = source; } } @@ -178,7 +173,7 @@ private synchronized void downloadNext() { if (activeDownloadsPerSource.getOrDefault(source, 0) < MAX_DOWNLOADS_IN_PROGRESS) { bestKey = entry.getKey(); bestTask = task; - bestDomain = source; + bestSource = source; break; } } @@ -188,11 +183,13 @@ private synchronized void downloadNext() { // --- EXECUTE --- queuedDownloads.remove(bestKey); - activeDownloadsPerSource.merge(bestDomain, 1, Integer::sum); + activeDownloadsPerSource.merge(bestSource, 1, Integer::sum); final FileInspection.HashPathPair key = bestKey; final QueuedDownload task = bestTask; - final String activeDomain = bestDomain; + final String activeDomain = bestSource; + + LOGGER.info("Queuning download for: {} {} {}", task.file, task.fileSize, activeDomain); CompletableFuture future = CompletableFuture.runAsync(() -> { try { @@ -264,9 +261,9 @@ private boolean attemptDownload(FileInspection.HashPathPair hashPathPair, Queued try { if (url != null && !Objects.equals(url, "host") && task.attempts < MAX_DOWNLOAD_ATTEMPTS * numberOfIndexes) { - httpDownloadFile(url, tempStoreFile); + httpDownloader.download(url, tempStoreFile, this::updateNetworkProgress); } else if (downloadClient != null) { - hostDownloadFile(hashPathPair, tempStoreFile); + hostDownloadFile(hashPathPair, tempStoreFile, this::updateNetworkProgress); } else { return false; } @@ -335,42 +332,12 @@ private void handleRetry(FileInspection.HashPathPair key, QueuedDownload task, b } } - // --- IO --- - - private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, Path targetFile) throws IOException { + private void hostDownloadFile(FileInspection.HashPathPair hashPathPair, Path targetFile, IntConsumer progressAction) throws IOException { SmartFileUtils.createParentDirs(targetFile); - var future = downloadClient.downloadFile(hashPathPair.hash().getBytes(StandardCharsets.UTF_8), targetFile, this::updateNetworkProgress); + var future = downloadClient.downloadFile(hashPathPair.hash().getBytes(StandardCharsets.UTF_8), targetFile, progressAction); future.join(); } - private void httpDownloadFile(String urlString, Path targetFile) throws IOException, InterruptedException { - SmartFileUtils.createParentDirs(targetFile); - URLConnection connection = getHttpConnection(urlString); - - try (InputStream rawIn = connection.getInputStream(); - InputStream in = "gzip".equals(connection.getHeaderField("Content-Encoding")) ? new GZIPInputStream(rawIn) : rawIn; - OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile.toFile()), 64 * 1024)) { - - byte[] buffer = new byte[NetUtils.DEFAULT_CHUNK_SIZE]; - int bytesRead; - while ((bytesRead = in.read(buffer)) != -1) { - if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Download cancelled"); - out.write(buffer, 0, bytesRead); - updateNetworkProgress(bytesRead); - } - } - } - - private URLConnection getHttpConnection(String urlString) throws IOException { - URL url = new URL(urlString); - URLConnection connection = url.openConnection(); - connection.addRequestProperty("Accept-Encoding", "gzip"); - connection.addRequestProperty("User-Agent", "github/skidamek/automodpack/" + AM_VERSION); - connection.setConnectTimeout(10000); - connection.setReadTimeout(10000); - return connection; - } - private void updateNetworkProgress(long bytes) { totalBytesDownloaded.addAndGet(bytes); speedometer.addBytes(bytes); diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/HttpFileDownloader.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/HttpFileDownloader.java new file mode 100644 index 000000000..eaa8bbec2 --- /dev/null +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/HttpFileDownloader.java @@ -0,0 +1,90 @@ +package pl.skidam.automodpack_loader_core.utils; + +import pl.skidam.automodpack_core.protocol.NetUtils; +import pl.skidam.automodpack_core.utils.SmartFileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.*; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.Executors; +import java.util.function.IntConsumer; +import java.util.zip.GZIPInputStream; + +import static pl.skidam.automodpack_core.Constants.AM_VERSION; + +public class HttpFileDownloader { + + private static final Logger LOGGER = LogManager.getLogger(); + + // Shared Client for HTTP/2 Multiplexing and Connection Pooling + private static final HttpClient CLIENT = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) // Auto-negotiate HTTP/2 + .followRedirects(HttpClient.Redirect.NORMAL) + .connectTimeout(Duration.ofSeconds(10)) + .executor(Executors.newCachedThreadPool()) // Async handler + .build(); + + /** + * Downloads a file from a URL to a target path using HTTP/2 if available. + * Blocks the calling thread (designed for use in Worker Threads). + * + * @param url The source URL. + * @param target The destination file path. + * @param progressAction A callback to report bytes read (for bandwidth tracking). + * @throws IOException If network or IO fails. + * @throws InterruptedException If the download is cancelled. + */ + public void download(String url, Path target, IntConsumer progressAction) throws IOException, InterruptedException { + SmartFileUtils.createParentDirs(target); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("User-Agent", "github/skidamek/automodpack/" + AM_VERSION) + .header("Accept-Encoding", "gzip") + .timeout(Duration.ofSeconds(10)) + .GET() + .build(); + + try { + HttpResponse response = CLIENT.send(request, HttpResponse.BodyHandlers.ofInputStream()); + + LOGGER.info("HTTPS Download {}: Source={} Protocol={} Status={}", target.getFileName(), url, response.version(), response.statusCode()); + + if (response.statusCode() != 200) { + throw new IOException("HTTP Error " + response.statusCode() + " for " + url); + } + + boolean isGzip = "gzip".equalsIgnoreCase(response.headers().firstValue("Content-Encoding").orElse("")); + + try (InputStream rawIn = response.body(); + InputStream in = isGzip ? new GZIPInputStream(rawIn) : rawIn; + OutputStream out = new BufferedOutputStream(new FileOutputStream(target.toFile()), 64 * 1024)) { + + byte[] buffer = new byte[NetUtils.DEFAULT_CHUNK_SIZE]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + if (Thread.currentThread().isInterrupted()) throw new InterruptedException(); + out.write(buffer, 0, bytesRead); + + if (progressAction != null) { + progressAction.accept(bytesRead); + } + } + } + + } catch (InterruptedException e) { + throw e; // Rethrow to handle cancellation + } catch (IOException e) { + throw e; // Rethrow IO errors + } catch (Exception e) { + // Wrap unexpected HTTP client errors + throw new IOException("HTTP Client Protocol Error", e); + } + } +} \ No newline at end of file From f7e7f12b8a30870485b8d1a044accf2b115aed3a Mon Sep 17 00:00:00 2001 From: skidam Date: Mon, 9 Feb 2026 20:15:51 +0100 Subject: [PATCH 22/22] Improve speedometer a little --- .../automodpack_loader_core/utils/SpeedFormatter.java | 2 +- .../skidam/automodpack_loader_core/utils/Speedometer.java | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java index c70bf268f..b4b54374f 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/SpeedFormatter.java @@ -18,7 +18,7 @@ public static String formatSpeed(long bytesPerSec) { } public static String formatETA(long seconds) { - if (seconds <= 0) return "-1"; + if (seconds < 0) return "-1"; long days = seconds / 86400; long hours = (seconds % 86400) / 3600; diff --git a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java index 41759dd7f..944df409c 100644 --- a/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java +++ b/loader/core/src/main/java/pl/skidam/automodpack_loader_core/utils/Speedometer.java @@ -77,15 +77,17 @@ public synchronized long getSpeed() { } public synchronized long getETA() { - long remainingBytes = Math.max(0, totalBytesExpected.get() - totalBytesReceived.get()); + long receievedBytes = totalBytesReceived.get(); + if (receievedBytes <= 0) return -1; + long remainingBytes = Math.max(0, totalBytesExpected.get() - receievedBytes); if (remainingBytes == 0) return 0; // Get the "Real" speed from the window (not the smoothed one) - double realSpeed = calculateWindowSpeed(System.currentTimeMillis(), totalBytesReceived.get()); + double realSpeed = calculateWindowSpeed(System.currentTimeMillis(), receievedBytes); // Safety: If speed is 0 or very low, return -1 (unknown) if (realSpeed < 1024) return -1; - return (long) (remainingBytes / realSpeed); + return (long) (remainingBytes / realSpeed) + 1; } } \ No newline at end of file