From 8e1805fda97c104aed46d3d7536e78dc4627c296 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Sun, 28 Dec 2025 19:59:02 +0800 Subject: [PATCH 1/3] Improves Zstd and KeyValue stream handling Refactors Zstd decompression to use streaming to avoid long JNI critical locks, improving memory management. Removes unnecessary buffering from KeyValue file operations. --- .../in/dragonbra/javasteam/types/KeyValue.kt | 6 ++-- .../in/dragonbra/javasteam/util/VZipUtil.kt | 14 ++------ .../in/dragonbra/javasteam/util/VZstdUtil.kt | 32 +++++++++++++++---- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt b/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt index 68c03ecd..eba20e27 100644 --- a/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt +++ b/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt @@ -323,7 +323,7 @@ class KeyValue @JvmOverloads constructor( * @param asBinary If set to true, saves this instance as binary. */ fun saveToFile(file: File, asBinary: Boolean) { - FileOutputStream(file, false).buffered().use { f -> saveToStream(f, asBinary) } + FileOutputStream(file, false).use { f -> saveToStream(f, asBinary) } } /** @@ -332,7 +332,7 @@ class KeyValue @JvmOverloads constructor( * @param asBinary If set to true, saves this instance as binary. */ fun saveToFile(path: String, asBinary: Boolean) { - FileOutputStream(path, false).buffered().use { f -> saveToStream(f, asBinary) } + FileOutputStream(path, false).use { f -> saveToStream(f, asBinary) } } /** @@ -473,7 +473,7 @@ class KeyValue @JvmOverloads constructor( } try { - FileInputStream(file).buffered(8192).use { input -> + FileInputStream(file).use { input -> val kv = KeyValue() if (asBinary) { diff --git a/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt b/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt index fa2d84e4..fa705a3a 100644 --- a/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt +++ b/src/main/java/in/dragonbra/javasteam/util/VZipUtil.kt @@ -6,7 +6,7 @@ import `in`.dragonbra.javasteam.util.stream.BinaryReader import `in`.dragonbra.javasteam.util.stream.MemoryStream import `in`.dragonbra.javasteam.util.stream.SeekOrigin import org.tukaani.xz.LZMAInputStream -import java.util.zip.* +import java.util.zip.DataFormatException import kotlin.math.max @Suppress("SpellCheckingInspection", "unused") @@ -21,11 +21,6 @@ object VZipUtil { private const val VERSION: Byte = 'a'.code.toByte() - // Thread-local window buffer pool to avoid repeated allocations - private val windowBufferPool = ThreadLocal.withInitial { - ByteArray(1 shl 23) // 8MB max size - } - @JvmStatic fun decompress(ms: MemoryStream, destination: ByteArray, verifyChecksum: Boolean = true): Int { try { @@ -67,12 +62,7 @@ object VZipUtil { // If the value of dictionary size in properties is smaller than (1 << 12), // the LZMA decoder must set the dictionary size variable to (1 << 12). - val windowSize = max(1 shl 12, dictionarySize) - val windowBuffer = if (windowSize <= (1 shl 23)) { - windowBufferPool.get() // Reuse thread-local buffer - } else { - ByteArray(windowSize) // Fallback for unusually large windows - } + val windowBuffer = ByteArray(max(1 shl 12, dictionarySize)) val bytesRead = LZMAInputStream( ms, sizeDecompressed.toLong(), diff --git a/src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt b/src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt index ee3ea888..20dd000e 100644 --- a/src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt +++ b/src/main/java/in/dragonbra/javasteam/util/VZstdUtil.kt @@ -1,7 +1,8 @@ package `in`.dragonbra.javasteam.util -import com.github.luben.zstd.Zstd +import com.github.luben.zstd.ZstdInputStream import `in`.dragonbra.javasteam.util.log.LogManager +import java.io.ByteArrayInputStream import java.io.IOException import java.nio.ByteBuffer import java.nio.ByteOrder @@ -9,6 +10,7 @@ import java.nio.ByteOrder object VZstdUtil { private const val VZSTD_HEADER: Int = 0x615A5356 + private const val STREAM_CHUNK_SIZE = 64 * 1024 // 64KB chunks private const val HEADER_SIZE = 8 private const val FOOTER_SIZE = 15 @@ -53,13 +55,31 @@ object VZstdUtil { throw IllegalArgumentException("The destination buffer is smaller than the decompressed data size.") } - val compressedData = buffer.copyOfRange(HEADER_SIZE, buffer.size - FOOTER_SIZE) // :( allocations - try { - val bytesDecompressed = Zstd.decompress(destination, compressedData) + // Use streaming decompression to avoid long JNI critical locks + var totalDecompressed = 0 + + // Use direct ByteArrayInputStream with offset to avoid copying compressed data + ByteArrayInputStream(buffer, HEADER_SIZE, buffer.size - FOOTER_SIZE).use { byteStream -> + ZstdInputStream(byteStream).use { zstdStream -> + var bytesRead: Int + var offset = 0 + + // Read in chunks to break up JNI locks + while (offset < sizeDecompressed) { + val toRead = minOf(STREAM_CHUNK_SIZE, sizeDecompressed - offset) + bytesRead = zstdStream.read(destination, offset, toRead) + + if (bytesRead == -1) break + + offset += bytesRead + totalDecompressed += bytesRead + } + } + } - if (bytesDecompressed != sizeDecompressed.toLong()) { - throw IOException("Failed to decompress Zstd (expected $sizeDecompressed bytes, got $bytesDecompressed).") + if (totalDecompressed != sizeDecompressed) { + throw IOException("Failed to decompress Zstd (expected $sizeDecompressed bytes, got $totalDecompressed).") } if (verifyChecksum) { From c54b018d9512c2e4701124039a2a97bb2905969b Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Sun, 28 Dec 2025 20:37:30 +0800 Subject: [PATCH 2/3] Revert KeyValue.kt --- src/main/java/in/dragonbra/javasteam/types/KeyValue.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt b/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt index eba20e27..68c03ecd 100644 --- a/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt +++ b/src/main/java/in/dragonbra/javasteam/types/KeyValue.kt @@ -323,7 +323,7 @@ class KeyValue @JvmOverloads constructor( * @param asBinary If set to true, saves this instance as binary. */ fun saveToFile(file: File, asBinary: Boolean) { - FileOutputStream(file, false).use { f -> saveToStream(f, asBinary) } + FileOutputStream(file, false).buffered().use { f -> saveToStream(f, asBinary) } } /** @@ -332,7 +332,7 @@ class KeyValue @JvmOverloads constructor( * @param asBinary If set to true, saves this instance as binary. */ fun saveToFile(path: String, asBinary: Boolean) { - FileOutputStream(path, false).use { f -> saveToStream(f, asBinary) } + FileOutputStream(path, false).buffered().use { f -> saveToStream(f, asBinary) } } /** @@ -473,7 +473,7 @@ class KeyValue @JvmOverloads constructor( } try { - FileInputStream(file).use { input -> + FileInputStream(file).buffered(8192).use { input -> val kv = KeyValue() if (asBinary) { From d909a44390b0629baa53619c4097afd9b8398c7b Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 29 Dec 2025 10:03:36 -0600 Subject: [PATCH 3/3] Bump version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 7c2a0dcd..24cfb08b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -20,7 +20,7 @@ plugins { allprojects { group = "in.dragonbra" - version = "1.8.0" + version = "1.8.1-SNAPSHOT" } repositories {