From 5be45beac0987ee31b0154fcc166e508afff47f6 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 10:53:59 -0500 Subject: [PATCH 01/44] Initial push of a new Depot Downloader --- .gitignore | 2 + build.gradle.kts | 2 +- gradle/libs.versions.toml | 7 + javasteam-depotdownloader/.gitignore | 1 + javasteam-depotdownloader/build.gradle.kts | 80 + .../depotdownloader/CDNClientPool.kt | 140 ++ .../depotdownloader/ContentDownloader.kt | 1889 +++++++++++++++++ .../ContentDownloaderException.kt | 7 + .../depotdownloader/DepotConfigStore.kt | 53 + .../javasteam/depotdownloader/HttpClient.kt | 46 + .../depotdownloader/IDownloadListener.kt | 35 + .../depotdownloader/Steam3Session.kt | 340 +++ .../javasteam/depotdownloader/Util.kt | 207 ++ .../depotdownloader/data/ChunkMatch.kt | 9 + .../depotdownloader/data/DepotDownloadInfo.kt | 41 + .../depotdownloader/data/DepotFilesData.kt | 20 + .../depotdownloader/data/DownloadCounters.kt | 26 + .../depotdownloader/data/DownloadItems.kt | 62 + .../depotdownloader/data/DownloadProgress.kt | 72 + .../depotdownloader/data/FileStreamData.kt | 16 + javasteam-samples/build.gradle.kts | 3 +- .../_023_downloadapp/SampleDownloadApp.java | 323 ++- settings.gradle.kts | 7 +- .../dragonbra/javasteam/steam/cdn/Client.kt | 367 ++-- .../javasteam/steam/cdn/ClientLancache.kt | 128 +- .../javasteam/steam/cdn/ClientPool.kt | 142 -- .../javasteam/steam/cdn/DepotChunk.kt | 7 +- .../dragonbra/javasteam/steam/cdn/Server.kt | 72 +- .../steam/contentdownloader/ChunkMatch.kt | 8 - .../contentdownloader/ContentDownloader.kt | 724 ------- .../contentdownloader/DepotDownloadCounter.kt | 8 - .../contentdownloader/DepotDownloadInfo.kt | 11 - .../steam/contentdownloader/DepotFilesData.kt | 11 - .../contentdownloader/FileManifestProvider.kt | 210 -- .../steam/contentdownloader/FileStreamData.kt | 10 - .../GlobalDownloadCounter.kt | 6 - .../contentdownloader/IManifestProvider.kt | 36 - .../MemoryManifestProvider.kt | 28 - .../contentdownloader/ProgressCallback.kt | 8 - .../steamcontent/CDNAuthToken.kt} | 6 +- .../handlers/steamcontent/SteamContent.kt | 14 +- .../ISteamConfigurationBuilder.kt | 9 - .../configuration/SteamConfiguration.kt | 7 - .../SteamConfigurationBuilder.kt | 8 - .../configuration/SteamConfigurationState.kt | 2 - .../webapi/ContentServerDirectoryService.kt | 33 +- .../in/dragonbra/javasteam/types/PubFile.kt | 108 + .../in/dragonbra/javasteam/util/Utils.java | 36 - .../javasteam/steam/cdn/CDNClientTest.java | 10 +- 49 files changed, 3788 insertions(+), 1609 deletions(-) create mode 100644 javasteam-depotdownloader/.gitignore create mode 100644 javasteam-depotdownloader/build.gradle.kts create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt rename src/main/java/in/dragonbra/javasteam/steam/{cdn/AuthToken.kt => handlers/steamcontent/CDNAuthToken.kt} (74%) create mode 100644 src/main/java/in/dragonbra/javasteam/types/PubFile.kt diff --git a/.gitignore b/.gitignore index 2f54edb3..781bc548 100644 --- a/.gitignore +++ b/.gitignore @@ -75,7 +75,9 @@ loginkey.txt sentry.bin server_list.bin /steamapps/ +/depots/ /userfiles/ +refreshtoken.txt # Kotlin 2.0 /.kotlin/sessions/ diff --git a/build.gradle.kts b/build.gradle.kts index f8202d81..fcc9e504 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -150,10 +150,10 @@ dependencies { mockitoAgent(libs.test.mock.core) { isTransitive = false } implementation(libs.bundles.ktor) + implementation(libs.bundles.okHttp) implementation(libs.commons.lang3) implementation(libs.kotlin.coroutines) implementation(libs.kotlin.stdib) - implementation(libs.okHttp) implementation(libs.protobuf.java) compileOnly(libs.xz) compileOnly(libs.zstd) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a878e273..edaaa11a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -41,6 +41,7 @@ ktor-client-cio = { module = "io.ktor:ktor-client-cio", version.ref = "ktor" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-websocket = { module = "io.ktor:ktor-client-websockets", version.ref = "ktor" } okHttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okHttp" } +okHttp-coroutines = { module = "com.squareup.okhttp3:okhttp-coroutines", version.ref = "okHttp" } protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } protobuf-protoc = { module = "com.google.protobuf:protoc", version.ref = "protobuf" } xz = { module = "org.tukaani:xz", version.ref = "xz" } @@ -65,6 +66,7 @@ qrCode = { module = "pro.leaco.qrcode:console-qrcode", version.ref = "qrCode" } kotlin-dokka = { id = "org.jetbrains.dokka", version.ref = "dokka" } kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } kotlin-kotlinter = { id = "org.jmailen.kotlinter", version.ref = "kotlinter" } +kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } maven-publish = { id = "io.github.gradle-nexus.publish-plugin", version.ref = "publishPlugin" } protobuf-gradle = { id = "com.google.protobuf", version.ref = "protobuf-gradle" } @@ -86,3 +88,8 @@ ktor = [ "ktor-client-cio", "ktor-client-websocket", ] + +okHttp = [ + "okHttp", + "okHttp-coroutines", +] diff --git a/javasteam-depotdownloader/.gitignore b/javasteam-depotdownloader/.gitignore new file mode 100644 index 00000000..d1638636 --- /dev/null +++ b/javasteam-depotdownloader/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/javasteam-depotdownloader/build.gradle.kts b/javasteam-depotdownloader/build.gradle.kts new file mode 100644 index 00000000..5ad11c36 --- /dev/null +++ b/javasteam-depotdownloader/build.gradle.kts @@ -0,0 +1,80 @@ +import org.gradle.kotlin.dsl.withType +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jmailen.gradle.kotlinter.tasks.FormatTask +import org.jmailen.gradle.kotlinter.tasks.LintTask + +plugins { + alias(libs.plugins.kotlin.dokka) + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.kotlin.kotlinter) + alias(libs.plugins.kotlin.serialization) + alias(libs.plugins.protobuf.gradle) + id("maven-publish") + id("signing") +} + +repositories { + mavenCentral() +} + +kotlin { + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget(libs.versions.java.get())) + } +} + +/* Protobufs */ +protobuf.protoc { + artifact = libs.protobuf.protoc.get().toString() +} + + +java { + sourceCompatibility = JavaVersion.toVersion(libs.versions.java.get()) + targetCompatibility = JavaVersion.toVersion(libs.versions.java.get()) + withSourcesJar() +} + +/* Java-Kotlin Docs */ +dokka { + moduleName.set("JavaSteam") + dokkaSourceSets.main { + suppressGeneratedFiles.set(false) // Allow generated files to be documented. + perPackageOption { + // Deny most of the generated files. + matchingRegex.set("in.dragonbra.javasteam.(protobufs|enums|generated).*") + suppress.set(true) + } + } +} + +// Make sure Maven Publishing gets javadoc +val javadocJar by tasks.registering(Jar::class) { + dependsOn(tasks.dokkaGenerate) + archiveClassifier.set("javadoc") + from(layout.buildDirectory.dir("dokka/html")) +} +artifacts { + archives(javadocJar) +} + +/* Kotlinter */ +tasks.withType { + this.source = this.source.minus(fileTree("build/generated")).asFileTree +} +tasks.withType { + this.source = this.source.minus(fileTree("build/generated")).asFileTree +} + +dependencies { + implementation(rootProject) // TODO verify if this causes something like a circular dependency. + + implementation("com.squareup.okio:okio:3.16.0") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + + implementation(libs.bundles.ktor) + implementation(libs.commons.lang3) + implementation(libs.kotlin.coroutines) + implementation(libs.kotlin.stdib) + implementation(libs.protobuf.java) +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt new file mode 100644 index 00000000..6ea50137 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -0,0 +1,140 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.steam.cdn.Client +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlin.jvm.Throws + +/** + * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. + * @param steamClient an instance of [SteamClient] + * @param appId the selected app id to ensure an endpoint supports the download. + * @param scope (optional) the [CoroutineScope] to use. + * @param debug enable or disable logging through [LogManager] + * + * @author Oxters + * @author Lossy + * @since Nov 7, 2024 + */ +class CDNClientPool( + private val steamClient: SteamClient, + private val appId: Int, + private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), + debug: Boolean = false, +) : AutoCloseable { + + companion object { + fun init( + steamClient: SteamClient, + appId: Int, + debug: Boolean, + ): CDNClientPool = CDNClientPool(steamClient = steamClient, appId = appId, debug = debug) + } + + private var logger: Logger? = null + + private val servers: ArrayList = arrayListOf() + + private var nextServer: Int = 0 + + private val mutex: Mutex = Mutex() + + var cdnClient: Client? = null + private set + + var proxyServer: Server? = null + private set + + init { + cdnClient = Client(steamClient) + + if (debug) { + logger = LogManager.getLogger(CDNClientPool::class.java) + } + } + + override fun close() { + scope.cancel() + + servers.clear() + + cdnClient = null + proxyServer = null + + logger = null + } + + @Throws(Exception::class) + suspend fun updateServerList(maxNumServers: Int? = null) = mutex.withLock { + if (servers.isNotEmpty()) { + servers.clear() + } + + val serversForSteamPipe = steamClient.getHandler()!!.getServersForSteamPipe( + cellId = steamClient.cellID ?: 0, + maxNumServers = maxNumServers, + parentScope = scope + ).await() + + proxyServer = serversForSteamPipe.firstOrNull { it.useAsProxy } + + val weightedCdnServers = serversForSteamPipe + .filter { server -> + val isEligibleForApp = server.allowedAppIds.isEmpty() || server.allowedAppIds.contains(appId) + isEligibleForApp && (server.type == "SteamCache" || server.type == "CDN") + } + .sortedBy { it.weightedLoad } + + // ContentServerPenalty removed for now. + + servers.addAll(weightedCdnServers) + + // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } + logger?.debug("Found ${servers.size} Servers: \n") + + if (servers.isEmpty()) { + throw Exception("Failed to retrieve any download servers.") + } + } + + suspend fun getConnection(): Server = mutex.withLock { + val server = servers[nextServer % servers.count()] + + logger?.debug("Getting connection $server") + + return server + } + + suspend fun returnConnection(server: Server?) = mutex.withLock { + if (server == null) { + return@withLock + } + + logger?.debug("Returning connection: $server") + + // nothing to do, maybe remove from ContentServerPenalty? + } + + suspend fun returnBrokenConnection(server: Server?) = mutex.withLock { + if (server == null) { + return@withLock + } + + logger?.debug("Returning broken connection: $server") + + if (servers[nextServer % servers.count()] == server) { + nextServer++ + + // TODO: Add server to ContentServerPenalty + } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt new file mode 100644 index 00000000..2571e2c0 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -0,0 +1,1889 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.AppItem +import `in`.dragonbra.javasteam.depotdownloader.data.ChunkMatch +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadCounter +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo +import `in`.dragonbra.javasteam.depotdownloader.data.DepotFilesData +import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadStatus +import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress +import `in`.dragonbra.javasteam.depotdownloader.data.FileStreamData +import `in`.dragonbra.javasteam.depotdownloader.data.GlobalDownloadCounter +import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress +import `in`.dragonbra.javasteam.depotdownloader.data.PubFileItem +import `in`.dragonbra.javasteam.depotdownloader.data.UgcItem +import `in`.dragonbra.javasteam.enums.EAccountType +import `in`.dragonbra.javasteam.enums.EAppInfoSection +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.steam.cdn.ClientLancache +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamapps.License +import `in`.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.callback.UGCDetailsCallback +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.types.FileData +import `in`.dragonbra.javasteam.types.KeyValue +import `in`.dragonbra.javasteam.types.PublishedFileID +import `in`.dragonbra.javasteam.types.UGCHandle +import `in`.dragonbra.javasteam.util.Adler32 +import `in`.dragonbra.javasteam.util.SteamKitWebRequestException +import `in`.dragonbra.javasteam.util.Strings +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsChannel +import io.ktor.http.HttpHeaders +import io.ktor.utils.io.core.readAvailable +import io.ktor.utils.io.core.remaining +import io.ktor.utils.io.readRemaining +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.future.future +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.withContext +import kotlinx.coroutines.yield +import okio.Buffer +import okio.FileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer +import org.apache.commons.lang3.SystemUtils +import java.io.Closeable +import java.io.IOException +import java.lang.IllegalStateException +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.* +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import kotlin.collections.mutableListOf +import kotlin.collections.set +import kotlin.text.toLongOrNull + +/** + * [ContentDownloader] is a JavaSteam module that is able to download Games, Workshop Items, and other content from Steam. + * @param steamClient an instance of [SteamClient] + * @param licenses a list of licenses the logged-in user has. This is provided by [LicenseListCallback] + * @param debug enable or disable logging through [LogManager] + * @param useLanCache try and detect a local Steam Cache server. + * @param maxDownloads the number of simultaneous downloads. + * + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +@Suppress("unused") +class ContentDownloader @JvmOverloads constructor( + private val steamClient: SteamClient, + private val licenses: List, // To be provided from [LicenseListCallback] + private val debug: Boolean = false, // Enable debugging features, such as logging + private val useLanCache: Boolean = false, // Try and detect a lan cache server. + private var maxDownloads: Int = 8, // Max concurrent downloads +) : Closeable { + + companion object { + const val INVALID_APP_ID: Int = Int.MAX_VALUE + const val INVALID_DEPOT_ID: Int = Int.MAX_VALUE + const val INVALID_MANIFEST_ID: Long = Long.MAX_VALUE + + const val CONFIG_DIR: String = ".DepotDownloader" + const val DEFAULT_BRANCH: String = "public" + const val DEFAULT_DOWNLOAD_DIR: String = "depots" + + val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" + } + + // What is a PriorityQueue? + + private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } + + private val items = CopyOnWriteArrayList(ArrayList()) + private val listeners = CopyOnWriteArrayList() + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private var logger: Logger? = null + private val isStarted: AtomicBoolean = AtomicBoolean(false) + private val processingChannel = Channel(Channel.UNLIMITED) + private val remainingItems = AtomicInteger(0) + private val lastFileProgressUpdate = ConcurrentHashMap() + private val progressUpdateInterval = 500L // ms + + private var steam3: Steam3Session? = null + + private var cdnClientPool: CDNClientPool? = null + + private var config: Config = Config() + + // region [REGION] Private data classes. + + private data class NetworkChunkItem( + val fileStreamData: FileStreamData, + val fileData: FileData, + val chunk: ChunkData, + val totalChunksForFile: Int, + ) + + private data class DirectoryResult(val success: Boolean, val installDir: Path?) + + private data class Config( + val installPath: Path? = null, + val betaPassword: String? = null, + val downloadAllPlatforms: Boolean = false, + val downloadAllArchs: Boolean = false, + val downloadAllLanguages: Boolean = false, + val androidEmulation: Boolean = false, + val downloadManifestOnly: Boolean = false, + val installToGameNameDirectory: Boolean = false, + + // Not used yet in code + val usingFileList: Boolean = false, + var filesToDownloadRegex: List = emptyList(), + var filesToDownload: HashSet = hashSetOf(), + var verifyAll: Boolean = false, + ) + + // endregion + + init { + if (debug) { + logger = LogManager.getLogger(ContentDownloader::class.java) + } + + logger?.debug("DepotDownloader launched with ${licenses.size} for account") + + steam3 = Steam3Session(steamClient, debug) + } + + // region [REGION] Downloading Operations + @Throws(IllegalStateException::class) + suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { + val details = requireNotNull( + steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) + ) { "Pub File Null" } + + if (!details.fileUrl.isNullOrBlank()) { + downloadWebFile(appId, details.filename, details.fileUrl) + } else if (details.hcontentFile > 0) { + downloadApp( + appId = appId, + depotManifestIds = listOf(appId to details.hcontentFile), + branch = DEFAULT_BRANCH, + os = null, + arch = null, + language = null, + lv = false, + isUgc = true, + ) + } else { + logger?.error("Unable to locate manifest ID for published file $publishedFileId") + } + } + + suspend fun downloadUGC( + appId: Int, + ugcId: Long, + ) { + var details: UGCDetailsCallback? = null + + val steamUser = requireNotNull(steam3!!.steamUser) + val steamId = requireNotNull(steamUser.steamID) + + if (steamId.accountType != EAccountType.AnonUser) { + val ugcHandle = UGCHandle(ugcId) + details = steam3!!.getUGCDetails(ugcHandle) + } else { + logger?.error("Unable to query UGC details for $ugcId from an anonymous account") + } + + if (!details?.url.isNullOrBlank()) { + downloadWebFile(appId = appId, fileName = details.fileName, url = details.url) + } else { + downloadApp( + appId = appId, + depotManifestIds = listOf(appId to ugcId), + branch = DEFAULT_BRANCH, + os = null, + arch = null, + language = null, + lv = false, + isUgc = true, + ) + } + } + + @Throws(IllegalStateException::class, IOException::class) + suspend fun downloadWebFile(appId: Int, fileName: String, url: String) { + val (success, installDir) = createDirectories(appId, 0, appId) + + if (!success) { + logger?.debug("Error: Unable to create install directories!") + return + } + + val stagingDir = installDir!! / "staging" + val fileStagingPath = stagingDir / fileName + val fileFinalPath = installDir / fileName + + filesystem.createDirectories(fileFinalPath.parent!!) + filesystem.createDirectories(fileStagingPath.parent!!) + + HttpClient.httpClient.use { client -> + logger?.debug("Starting download of $fileName...") + + val response = client.get(url) + val channel = response.bodyAsChannel() + + val totalBytes = response.headers[HttpHeaders.ContentLength]?.toLongOrNull() + + logger?.debug("File size: ${totalBytes?.let { Util.formatBytes(it) } ?: "Unknown"}") + + filesystem.sink(fileStagingPath).buffer().use { sink -> + val buffer = Buffer() + while (!channel.isClosedForRead) { + val packet = channel.readRemaining(DEFAULT_BUFFER_SIZE.toLong()) + if (!packet.exhausted()) { + // Read from Ktor packet into Okio buffer + val bytesRead = packet.remaining.toInt() + val tempArray = ByteArray(bytesRead) + packet.readAvailable(tempArray) + buffer.write(tempArray) + + // Write from buffer to sink + sink.writeAll(buffer) + } + } + } + + logger?.debug("Download completed.") + } + + if (filesystem.exists(fileFinalPath)) { + logger?.debug("Deleting $fileFinalPath") + filesystem.delete(fileFinalPath) + } + + try { + filesystem.atomicMove(fileStagingPath, fileFinalPath) + logger?.debug("File '$fileStagingPath' moved to final location: $fileFinalPath") + } catch (e: IOException) { + throw e + } + } + + // L4D2 (app) supports LV + @Throws(IllegalStateException::class) + suspend fun downloadApp( + appId: Int, + depotManifestIds: List>, + branch: String, + os: String?, + arch: String?, + language: String?, + lv: Boolean, + isUgc: Boolean, + ) { + var depotManifestIds = depotManifestIds.toMutableList() + + val steamUser = requireNotNull(steam3!!.steamUser) + cdnClientPool = CDNClientPool.init(steamClient, appId, debug) + + // Load our configuration data containing the depots currently installed + var configPath = config.installPath + if (configPath == null) { + configPath = DEFAULT_DOWNLOAD_DIR.toPath() + } + + filesystem.createDirectories(configPath) + DepotConfigStore.loadFromFile(configPath / CONFIG_DIR / "depot.config") + + steam3!!.requestAppInfo(appId) + + if (!accountHasAccess(appId, appId)) { + if (steamUser.steamID!!.accountType != EAccountType.AnonUser && steam3!!.requestFreeAppLicense(appId)) { + logger?.debug("Obtained FreeOnDemand license for app $appId") + + // Fetch app info again in case we didn't get it fully without a license. + steam3!!.requestAppInfo(appId, true) + } else { + val contentName = getAppName(appId) + throw ContentDownloaderException("App $appId ($contentName) is not available from this account.") + } + } + + val hasSpecificDepots = depotManifestIds.isNotEmpty() + val depotIdsFound = mutableListOf() + val depotIdsExpected = depotManifestIds.map { x -> x.first }.toMutableList() + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) + + if (isUgc) { + val workshopDepot = depots!!["workshopdepot"].asInteger() + if (workshopDepot != 0 && !depotIdsExpected.contains(workshopDepot)) { + depotIdsExpected.add(workshopDepot) + depotManifestIds = depotManifestIds.map { pair -> workshopDepot to pair.second }.toMutableList() + } + + depotIdsFound.addAll(depotIdsExpected) + } else { + logger?.debug("Using app branch: $branch") + + depots?.children?.forEach { depotSection -> + @Suppress("VariableInitializerIsRedundant") + var id = INVALID_DEPOT_ID + + if (depotSection.children.isEmpty()) { + return@forEach + } + + id = depotSection.name?.toIntOrNull() ?: return@forEach + + if (hasSpecificDepots && !depotIdsExpected.contains(id)) { + return@forEach + } + + if (!hasSpecificDepots) { + val depotConfig = depotSection["config"] + if (depotConfig != KeyValue.INVALID) { + if (!config.downloadAllPlatforms && + depotConfig["oslist"] != KeyValue.INVALID && + !depotConfig["oslist"].value.isNullOrBlank() + ) { + val osList = depotConfig["oslist"].value!!.split(",") + if (osList.indexOf(os ?: Util.getSteamOS(config.androidEmulation)) == -1) { + return@forEach + } + } + + if (!config.downloadAllArchs && + depotConfig["osarch"] != KeyValue.INVALID && + !depotConfig["osarch"].value.isNullOrBlank() + ) { + val depotArch = depotConfig["osarch"].value + if (depotArch != (arch ?: Util.getSteamArch())) { + return@forEach + } + } + + if (!config.downloadAllLanguages && + depotConfig["language"] != KeyValue.INVALID && + !depotConfig["language"].value.isNullOrBlank() + ) { + val depotLang = depotConfig["language"].value + if (depotLang != (language ?: "english")) { + return@forEach + } + } + + if (!lv && + depotConfig["lowviolence"] != KeyValue.INVALID && + depotConfig["lowviolence"].asBoolean() + ) { + return@forEach + } + } + } + + depotIdsFound.add(id) + + if (!hasSpecificDepots) { + depotManifestIds.add(id to INVALID_MANIFEST_ID) + } + } + + if (depotManifestIds.isEmpty() && !hasSpecificDepots) { + throw ContentDownloaderException("Couldn't find any depots to download for app $appId") + } + + if (depotIdsFound.size < depotIdsExpected.size) { + val remainingDepotIds = depotIdsExpected.subtract(depotIdsFound.toSet()) + throw ContentDownloaderException("Depot ${remainingDepotIds.joinToString(", ")} not listed for app $appId") + } + } + + val infos = mutableListOf() + + depotManifestIds.forEach { (depotId, manifestId) -> + val info = getDepotInfo(depotId, appId, manifestId, branch) + if (info != null) { + infos.add(info) + } + } + + downloadSteam3(infos) + } + + @Throws(IllegalStateException::class) + private suspend fun getDepotInfo( + depotId: Int, + appId: Int, + manifestId: Long, + branch: String, + ): DepotDownloadInfo? { + var manifestId = manifestId + var branch = branch + + if (appId != INVALID_APP_ID) { + steam3!!.requestAppInfo(appId) + } + + if (!accountHasAccess(appId, depotId)) { + logger?.error("Depot $depotId is not available from this account.") + return null + } + + if (manifestId == INVALID_MANIFEST_ID) { + manifestId = getSteam3DepotManifest(depotId, appId, branch) + + if (manifestId == INVALID_MANIFEST_ID && !branch.equals(DEFAULT_BRANCH, true)) { + logger?.error("Warning: Depot $depotId does not have branch named \"$branch\". Trying $DEFAULT_BRANCH branch.") + branch = DEFAULT_BRANCH + manifestId = getSteam3DepotManifest(depotId, appId, branch) + } + + if (manifestId == INVALID_MANIFEST_ID) { + logger?.error("Depot $depotId missing public subsection or manifest section.") + return null + } + } + + steam3!!.requestDepotKey(depotId, appId) + + val depotKey = steam3!!.depotKeys[depotId] + if (depotKey == null) { + logger?.error("No valid depot key for $depotId, unable to download.") + return null + } + + val uVersion = getSteam3AppBuildNumber(appId, branch) + + val (success, installDir) = createDirectories(depotId, uVersion, appId) + if (!success) { + logger?.error("Error: Unable to create install directories!") + return null + } + + // For depots that are proxied through depotfromapp, we still need to resolve the proxy app id, unless the app is freetodownload + var containingAppId = appId + val proxyAppId = getSteam3DepotProxyAppId(depotId, appId) + if (proxyAppId != INVALID_APP_ID) { + val common = getSteam3AppSection(appId, EAppInfoSection.Common) + if (common == null || !common["FreeToDownload"].asBoolean()) { + containingAppId = proxyAppId + } + } + + return DepotDownloadInfo( + depotId = depotId, + appId = containingAppId, + manifestId = manifestId, + branch = branch, + installDir = installDir!!, + depotKey = depotKey + ) + } + + private suspend fun getSteam3DepotManifest( + depotId: Int, + appId: Int, + branch: String, + ): Long { + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) + var depotChild = depots?.get(depotId.toString()) ?: KeyValue.INVALID + + if (depotChild == KeyValue.INVALID) { + return INVALID_MANIFEST_ID + } + + // Shared depots can either provide manifests, or leave you relying on their parent app. + // It seems that with the latter, "sharedinstall" will exist (and equals 2 in the one existance I know of). + // Rather than relay on the unknown sharedinstall key, just look for manifests. Test cases: 111710, 346680. + if (depotChild["manifests"] == KeyValue.INVALID && depotChild["depotfromapp"] != KeyValue.INVALID) { + val otherAppId = depotChild["depotfromapp"].asInteger() + if (otherAppId == appId) { + // This shouldn't ever happen, but ya never know with Valve. Don't infinite loop. + logger?.error("App $appId, Depot $depotId has depotfromapp of $otherAppId!") + return INVALID_MANIFEST_ID + } + + steam3!!.requestAppInfo(otherAppId) + + return getSteam3DepotManifest(depotId, otherAppId, branch) + } + + var manifests = depotChild["manifests"] + + if (manifests.children.isEmpty()) { + return INVALID_MANIFEST_ID + } + + var node = manifests[branch]["gid"] + + // Non passworded branch, found the manifest + if (node.value != null) { + return node.value!!.toLong() + } + + // If we requested public branch, and it had no manifest, nothing to do + if (branch.equals(DEFAULT_BRANCH, true)) { + return INVALID_MANIFEST_ID + } + + // Either the branch just doesn't exist, or it has a password + if (config.betaPassword.isNullOrBlank()) { + logger?.error("Branch $branch for depot $depotId was not found, either it does not exist or it has a password.") + return INVALID_MANIFEST_ID + } + + if (!steam3!!.appBetaPasswords.containsKey(branch)) { + // Submit the password to Steam now to get encryption keys + steam3!!.checkAppBetaPassword(appId, config.betaPassword!!) + + if (!steam3!!.appBetaPasswords.containsKey(branch)) { + logger?.error("Error: Password was invalid for branch $branch (or the branch does not exist)") + return INVALID_MANIFEST_ID + } + } + + // Got the password, request private depot section + // TODO: (SK) We're probably repeating this request for every depot? + val privateDepotSection = steam3!!.getPrivateBetaDepotSection(appId, branch) + + // Now repeat the same code to get the manifest gid from depot section + depotChild = privateDepotSection[depotId.toString()] + + if (depotChild == KeyValue.INVALID) { + return INVALID_MANIFEST_ID + } + + manifests = depotChild["manifests"] + + if (manifests.children.isEmpty()) { + return INVALID_MANIFEST_ID + } + + node = manifests[branch]["gid"] + + if (node.value == null) { + return INVALID_MANIFEST_ID + } + + return node.value!!.toLong() + } + + private fun getSteam3AppBuildNumber(appId: Int, branch: String): Int { + if (appId == INVALID_APP_ID) { + return 0 + } + + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) ?: KeyValue.INVALID + val branches = depots["branches"] + val node = branches[branch] + + if (node == KeyValue.INVALID) { + return 0 + } + + val buildId = node["buildid"] + + if (buildId == KeyValue.INVALID) { + return 0 + } + + return buildId.value!!.toInt() + } + + private fun getSteam3DepotProxyAppId(depotId: Int, appId: Int): Int { + val depots = getSteam3AppSection(appId, EAppInfoSection.Depots) ?: KeyValue.INVALID + val depotChild = depots[depotId.toString()] + + if (depotChild == KeyValue.INVALID) { + return INVALID_APP_ID + } + + if (depotChild["depotfromapp"] == KeyValue.INVALID) { + return INVALID_APP_ID + } + + return depotChild["depotfromapp"].asInteger() + } + + @Throws(IllegalStateException::class) + private fun createDirectories(depotId: Int, depotVersion: Int, appId: Int = 0): DirectoryResult { + var installDir: Path? + try { + if (config.installPath?.toString().isNullOrBlank()) { + // Android Check + if (SystemUtils.IS_OS_ANDROID) { + // This should propagate up to the caller. + throw IllegalStateException("Android must have an installation directory set.") + } + + filesystem.createDirectories(DEFAULT_DOWNLOAD_DIR.toPath()) + + if (config.installToGameNameDirectory) { + val gameName = getAppName(appId) + + if (gameName.isBlank()) { + throw IOException("Game name is blank, cannot create directory") + } + + installDir = DEFAULT_DOWNLOAD_DIR.toPath() / gameName + + filesystem.createDirectories(installDir) + } else { + val depotPath = DEFAULT_DOWNLOAD_DIR.toPath() / depotId.toString() + filesystem.createDirectories(depotPath) + + installDir = depotPath / depotVersion.toString() + filesystem.createDirectories(installDir) + } + + filesystem.createDirectories(installDir / CONFIG_DIR) + filesystem.createDirectories(installDir / STAGING_DIR) + } else { + filesystem.createDirectories(config.installPath!!) + + if (config.installToGameNameDirectory) { + val gameName = getAppName(appId) + + if (gameName.isBlank()) { + throw IOException("Game name is blank, cannot create directory") + } + + installDir = config.installPath!! / gameName + + filesystem.createDirectories(installDir) + } else { + installDir = config.installPath!! + } + + filesystem.createDirectories(installDir / CONFIG_DIR) + filesystem.createDirectories(installDir / STAGING_DIR) + } + } catch (e: IOException) { + logger?.error(e) + return DirectoryResult(false, null) + } + + return DirectoryResult(true, installDir) + } + + private fun getAppName(appId: Int): String { + val info = getSteam3AppSection(appId, EAppInfoSection.Common) ?: KeyValue.INVALID + return info["name"].asString() ?: "" + } + + private fun getSteam3AppSection(appId: Int, section: EAppInfoSection): KeyValue? { + if (steam3 == null) { + return null + } + + if (steam3!!.appInfo.isEmpty()) { + return null + } + + val app = steam3!!.appInfo[appId] ?: return null + + val appInfo = app.keyValues + val sectionKey = when (section) { + EAppInfoSection.Common -> "common" + EAppInfoSection.Extended -> "extended" + EAppInfoSection.Config -> "config" + EAppInfoSection.Depots -> "depots" + else -> throw ContentDownloaderException("${section.name} not implemented") + } + + val sectionKV = appInfo.children.firstOrNull { c -> c.name == sectionKey } + return sectionKV + } + + private suspend fun accountHasAccess(appId: Int, depotId: Int): Boolean { + val steamUser = requireNotNull(steam3!!.steamUser) + val steamID = requireNotNull(steamUser.steamID) + + if (licenses.isEmpty() && steamID.accountType != EAccountType.AnonUser) { + return false + } + + val licenseQuery = arrayListOf() + if (steamID.accountType == EAccountType.AnonUser) { + licenseQuery.add(17906) + } else { + licenseQuery.addAll(licenses.map { it.packageID }.distinct()) + } + + steam3!!.requestPackageInfo(licenseQuery) + + licenseQuery.forEach { license -> + steam3!!.packageInfo[license]?.let { pkg -> + val appIds = pkg.keyValues["appids"].children.map { it.asInteger() } + val depotIds = pkg.keyValues["depotids"].children.map { it.asInteger() } + if (depotId in appIds) { + return true + } + if (depotId in depotIds) { + return true + } + } + } + + // Check if this app is free to download without a license + val info = getSteam3AppSection(appId, EAppInfoSection.Common) + + return info != null && info["FreeToDownload"].asBoolean() + } + + private suspend fun downloadSteam3(depots: List): Unit = coroutineScope { + cdnClientPool?.updateServerList() + + val downloadCounter = GlobalDownloadCounter() + val depotsToDownload = ArrayList(depots.size) + val allFileNamesAllDepots = hashSetOf() + + var completedDepots = 0 + + // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup + depots.forEach { depot -> + val depotFileData = processDepotManifestAndFiles(depot, downloadCounter) + + if (depotFileData != null) { + depotsToDownload.add(depotFileData) + allFileNamesAllDepots.union(depotFileData.allFileNames) + } + + ensureActive() + } + + // If we're about to write all the files to the same directory, we will need to first de-duplicate any files by path + // This is in last-depot-wins order, from Steam or the list of depots supplied by the user + if (config.installPath != null && depotsToDownload.isNotEmpty()) { + val claimedFileNames = mutableSetOf() + for (i in depotsToDownload.indices.reversed()) { + // For each depot, remove all files from the list that have been claimed by a later depot + depotsToDownload[i].filteredFiles.removeAll { file -> file.fileName in claimedFileNames } + claimedFileNames.addAll(depotsToDownload[i].allFileNames) + } + } + + depotsToDownload.forEach { depotFileData -> + downloadSteam3DepotFiles(downloadCounter, depotFileData, allFileNamesAllDepots) + + completedDepots++ + + val snapshot = synchronized(downloadCounter) { + OverallProgress( + currentItem = completedDepots, + totalItems = depotsToDownload.size, + totalBytesDownloaded = downloadCounter.totalBytesUncompressed, + totalBytesExpected = downloadCounter.completeDownloadSize, + status = DownloadStatus.DOWNLOADING + ) + } + + notifyListeners { listener -> + listener.onOverallProgress(progress = snapshot) + } + } + + logger?.debug( + "Total downloaded: ${downloadCounter.totalBytesCompressed} bytes " + + "(${downloadCounter.totalBytesUncompressed} bytes uncompressed) from ${depots.size} depots" + ) + } + + private suspend fun processDepotManifestAndFiles( + depot: DepotDownloadInfo, + downloadCounter: GlobalDownloadCounter, + ): DepotFilesData? = withContext(Dispatchers.IO) { + val depotCounter = DepotDownloadCounter() + + logger?.debug("Processing depot ${depot.depotId}") + + var oldManifest: DepotManifest? = null + + @Suppress("VariableInitializerIsRedundant") + var newManifest: DepotManifest? = null + + val configDir = depot.installDir / CONFIG_DIR + + @Suppress("VariableInitializerIsRedundant") + var lastManifestId = INVALID_MANIFEST_ID + lastManifestId = DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] ?: INVALID_MANIFEST_ID + + // In case we have an early exit, this will force equiv of verifyall next run. + DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = INVALID_MANIFEST_ID + DepotConfigStore.save() + + if (lastManifestId != INVALID_MANIFEST_ID) { + // We only have to show this warning if the old manifest ID was different + val badHashWarning = lastManifestId != depot.manifestId + oldManifest = Util.loadManifestFromFile(configDir, depot.depotId, lastManifestId, badHashWarning) + } + + if (lastManifestId == depot.manifestId && oldManifest != null) { + newManifest = oldManifest + logger?.debug("Already have manifest ${depot.manifestId} for depot ${depot.depotId}.") + } else { + newManifest = Util.loadManifestFromFile(configDir, depot.depotId, depot.manifestId, true) + + if (newManifest != null) { + logger?.debug("Already have manifest ${depot.manifestId} for depot ${depot.depotId}.") + } else { + logger?.debug("Downloading depot ${depot.depotId} manifest") + notifyListeners { it.onStatusUpdate("Downloading manifest for depot ${depot.depotId}") } + + var manifestRequestCode: ULong = 0U + var manifestRequestCodeExpiration = Instant.MIN + + do { + ensureActive() + + var connection: Server? = null + + try { + connection = cdnClientPool!!.getConnection() + + var cdnToken: String? = null + + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") + } + } + + val now = Instant.now() + + // In order to download this manifest, we need the current manifest request code + // The manifest request code is only valid for a specific period in time + if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { + manifestRequestCode = steam3!!.getDepotManifestRequestCode( + depotId = depot.depotId, + appId = depot.appId, + manifestId = depot.manifestId, + branch = depot.branch, + ) + + // This code will hopefully be valid for one period following the issuing period + manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) + + // If we could not get the manifest code, this is a fatal error + if (manifestRequestCode == 0UL) { + cancel("manifestRequestCode is 0UL") + } + } + + logger?.debug("Downloading manifest ${depot.manifestId} from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") + + newManifest = cdnClientPool!!.cdnClient!!.downloadManifest( + depotId = depot.depotId, + manifestId = depot.manifestId, + manifestRequestCode = manifestRequestCode, + server = connection, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) + + cdnClientPool!!.returnConnection(connection) + } catch (e: CancellationException) { + // logger?.error("Connection timeout downloading depot manifest ${depot.depotId} ${depot.manifestId}. Retrying.") + logger?.error(e) + break + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet + if (e.statusCode == 403 && !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host)) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + + cdnClientPool!!.returnConnection(connection) + + continue + } + + cdnClientPool!!.returnBrokenConnection(connection) + + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${depot.depotId} for depot manifest ${depot.manifestId} ${e.statusCode}. Aborting.") + break + } + + // NotFound + if (e.statusCode == 404) { + logger?.error("Encountered 404 for depot manifest ${depot.depotId} ${depot.manifestId}. Aborting.") + break + } + + logger?.error("Encountered error downloading depot manifest ${depot.depotId} ${depot.manifestId}: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered error downloading manifest for depot ${depot.depotId} ${depot.manifestId}: ${e.message}") + } + } while (newManifest == null) + + if (newManifest == null) { + logger?.error("\nUnable to download manifest ${depot.manifestId} for depot ${depot.depotId}") + cancel() + } + + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() + + Util.saveManifestToFile(configDir, newManifest!!) + } + } + + logger?.debug("Manifest ${depot.manifestId} (${newManifest.creationTime})") + + if (config.downloadManifestOnly) { + Util.dumpManifestToTextFile(depot, newManifest) + return@withContext null + } + + val stagingDir = depot.installDir / STAGING_DIR + + val filesAfterExclusions = coroutineScope { + newManifest.files.filter { file -> + async { testIsFileIncluded(file.fileName) }.await() + } + } + val allFileNames = HashSet(filesAfterExclusions.size) + + // Pre-process + filesAfterExclusions.forEach { file -> + allFileNames.add(file.fileName) + + val fileFinalPath = depot.installDir / file.fileName + val fileStagingPath = stagingDir / file.fileName + + if (file.flags.contains(EDepotFileFlag.Directory)) { + filesystem.createDirectories(fileFinalPath) + filesystem.createDirectories(fileStagingPath) + } else { + // Some manifests don't explicitly include all necessary directories + filesystem.createDirectories(fileFinalPath.parent!!) + filesystem.createDirectories(fileStagingPath.parent!!) + + downloadCounter.completeDownloadSize += file.totalSize + depotCounter.completeDownloadSize += file.totalSize + } + } + + return@withContext DepotFilesData( + depotDownloadInfo = depot, + depotCounter = depotCounter, + stagingDir = stagingDir, + manifest = newManifest, + previousManifest = oldManifest, + filteredFiles = filesAfterExclusions.toMutableList(), + allFileNames = allFileNames, + ) + } + + @OptIn(DelicateCoroutinesApi::class) + private suspend fun downloadSteam3DepotFiles( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + allFileNamesAllDepots: HashSet, + ) = withContext(Dispatchers.IO) { + val depot = depotFilesData.depotDownloadInfo + val depotCounter = depotFilesData.depotCounter + + logger?.debug("Downloading depot ${depot.depotId}") + + val files = depotFilesData.filteredFiles.filter { !it.flags.contains(EDepotFileFlag.Directory) } + val networkChunkQueue = Channel(Channel.UNLIMITED) + + try { + val filesCompleted = AtomicInteger(0) + val lastReportedProgress = AtomicInteger(0) + coroutineScope { + // First parallel loop - process files and enqueue chunks + files.map { file -> + async { + yield() // Does this matter if its before? + downloadSteam3DepotFile( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = file, + networkChunkQueue = networkChunkQueue + ) + + val completed = filesCompleted.incrementAndGet() + if (completed % 10 == 0 || completed == files.size) { + val snapshot = synchronized(depotCounter) { + DepotProgress( + depotId = depot.depotId, + filesCompleted = completed, + totalFiles = files.size, + bytesDownloaded = depotCounter.sizeDownloaded, + totalBytes = depotCounter.completeDownloadSize, + status = DownloadStatus.PREPARING // Changed from DOWNLOADING + ) + } + + val lastReported = lastReportedProgress.get() + if (completed > lastReported && + lastReportedProgress.compareAndSet( + lastReported, + completed + ) + ) { + notifyListeners { listener -> + listener.onDepotProgress(depot.depotId, snapshot) + } + } + } + } + }.awaitAll() + + // Close the channel to signal no more items will be added + networkChunkQueue.close() + + // After all files allocated, send one update showing preparation complete + val progressReporter = launch { + while (true) { + delay(1000) + val snapshot = synchronized(depotCounter) { + DepotProgress( + depotId = depot.depotId, + filesCompleted = files.size, + totalFiles = files.size, + bytesDownloaded = depotCounter.sizeDownloaded, + totalBytes = depotCounter.completeDownloadSize, + status = DownloadStatus.DOWNLOADING + ) + } + notifyListeners { listener -> + listener.onDepotProgress(depot.depotId, snapshot) + } + } + } + + // Second parallel loop - process chunks from queue + try { + List(maxDownloads) { + async { + for (item in networkChunkQueue) { + downloadSteam3DepotFileChunk( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = item.fileData, + fileStreamData = item.fileStreamData, + chunk = item.chunk + ) + } + } + }.awaitAll() + } finally { + progressReporter.cancel() + } + } + } finally { + if (!networkChunkQueue.isClosedForSend) { + networkChunkQueue.close() + } + } + + // Check for deleted files if updating the depot. + if (depotFilesData.previousManifest != null) { + val previousFilteredFiles = depotFilesData.previousManifest.files + .filter { testIsFileIncluded(it.fileName) } + .map { it.fileName } + .toHashSet() + + // Check if we are writing to a single output directory. If not, each depot folder is managed independently + if (config.installPath == null) { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names + previousFilteredFiles.removeAll(depotFilesData.allFileNames) + } else { + // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names across all depots being downloaded + previousFilteredFiles.removeAll(allFileNamesAllDepots) + } + + previousFilteredFiles.forEach { existingFileName -> + val fileFinalPath = depot.installDir / existingFileName + + if (!filesystem.exists(fileFinalPath)) { + return@forEach + } + + filesystem.delete(fileFinalPath) + logger?.debug("Deleted $fileFinalPath") + } + } + + DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = depot.manifestId + DepotConfigStore.save() + logger?.debug("Depot ${depot.depotId} - Downloaded ${depotCounter.depotBytesCompressed} bytes (${depotCounter.depotBytesUncompressed} bytes uncompressed)") + } + + private suspend fun downloadSteam3DepotFile( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + file: FileData, + networkChunkQueue: Channel, + ) = withContext(Dispatchers.IO) { + ensureActive() + + val depot = depotFilesData.depotDownloadInfo + val stagingDir = depotFilesData.stagingDir + val depotDownloadCounter = depotFilesData.depotCounter + val oldProtoManifest = depotFilesData.previousManifest + + var oldManifestFile: FileData? = null + if (oldProtoManifest != null) { + oldManifestFile = oldProtoManifest.files.singleOrNull { it.fileName == file.fileName } + } + + val fileFinalPath = depot.installDir / file.fileName + val fileStagingPath = stagingDir / file.fileName + + // This may still exist if the previous run exited before cleanup + if (filesystem.exists(fileStagingPath)) { + filesystem.delete(fileStagingPath) + } + + var neededChunks: MutableList? = null + val fileDidExist = filesystem.exists(fileFinalPath) + if (!fileDidExist) { + logger?.debug("Pre-allocating: $fileFinalPath") + notifyListeners { it.onStatusUpdate("Allocating file: ${file.fileName}") } + + // create new file. need all chunks + try { + filesystem.openReadWrite(fileFinalPath).use { handle -> + handle.resize(file.totalSize) + } + } catch (e: IOException) { + throw ContentDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") + } + + neededChunks = ArrayList(file.chunks) + } else { + // open existing + if (oldManifestFile != null) { + neededChunks = arrayListOf() + + val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) + if (config.verifyAll || !hashMatches) { + // we have a version of this file, but it doesn't fully match what we want + if (config.verifyAll) { + logger?.debug("Validating: $fileFinalPath") + } + + val matchingChunks = arrayListOf() + + file.chunks.forEach { chunk -> + val oldChunk = oldManifestFile.chunks.firstOrNull { c -> + c.chunkID.contentEquals(chunk.chunkID) + } + if (oldChunk != null) { + val chunkMatch = ChunkMatch(oldChunk, chunk) + matchingChunks.add(chunkMatch) + } else { + neededChunks.add(chunk) + } + } + + val orderedChunks = matchingChunks.sortedBy { x -> x.oldChunk.offset } + + val copyChunks = arrayListOf() + + filesystem.openReadOnly(fileFinalPath).use { handle -> + orderedChunks.forEach { match -> + // Read the chunk data into a byte array + val length = match.oldChunk.uncompressedLength + val buffer = ByteArray(length) + handle.read(match.oldChunk.offset, buffer, 0, length) + + // Calculate Adler32 checksum + val adler = Adler32.calculate(buffer) + + // Convert checksum to byte array for comparison + val checksumBytes = Buffer().apply { + writeIntLe(match.oldChunk.checksum) + }.readByteArray() + val calculatedChecksumBytes = Buffer().apply { + writeIntLe(adler) + }.readByteArray() + + if (!calculatedChecksumBytes.contentEquals(checksumBytes)) { + neededChunks.add(match.newChunk) + } else { + copyChunks.add(match) + } + } + } + + if (!hashMatches || neededChunks.isNotEmpty()) { + filesystem.atomicMove(fileFinalPath, fileStagingPath) + + try { + filesystem.openReadOnly(fileStagingPath).use { oldHandle -> + filesystem.openReadWrite(fileFinalPath).use { newHandle -> + try { + newHandle.resize(file.totalSize) + } catch (ex: IOException) { + throw ContentDownloaderException( + "Failed to resize file to expected size $fileFinalPath: ${ex.message}" + ) + } + + for (match in copyChunks) { + val tmp = ByteArray(match.oldChunk.uncompressedLength) + oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) + newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) + } + } + } + } catch (e: Exception) { + logger?.error(e) + } + + filesystem.delete(fileStagingPath) + } + } + } else { + // No old manifest or file not in old manifest. We must validate. + filesystem.openReadWrite(fileFinalPath).use { handle -> + val fileSize = filesystem.metadata(fileFinalPath).size ?: 0L + if (fileSize.toULong() != file.totalSize.toULong()) { + try { + handle.resize(file.totalSize) + } catch (ex: IOException) { + throw ContentDownloaderException( + "Failed to allocate file $fileFinalPath: ${ex.message}" + ) + } + } + + logger?.debug("Validating $fileFinalPath") + notifyListeners { it.onStatusUpdate("Validating: ${file.fileName}") } + + neededChunks = Util.validateSteam3FileChecksums( + handle = handle, + chunkData = file.chunks.sortedBy { it.offset } + ).toMutableList() + } + } + + if (neededChunks!!.isEmpty()) { + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += file.totalSize + + val percentage = + (depotDownloadCounter.sizeDownloaded / depotDownloadCounter.completeDownloadSize.toFloat()) * 100.0f + logger?.debug("%.2f%% %s".format(percentage, fileFinalPath)) + } + + synchronized(downloadCounter) { + downloadCounter.completeDownloadSize -= file.totalSize + } + + return@withContext + } + + val sizeOnDisk = file.totalSize - neededChunks!!.sumOf { it.uncompressedLength } + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += sizeOnDisk + } + + synchronized(downloadCounter) { + downloadCounter.completeDownloadSize -= sizeOnDisk + } + } + + val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) + if (fileIsExecutable && + (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) + ) { + fileFinalPath.toFile().setExecutable(true) + } else if (!fileIsExecutable && + oldManifestFile != null && + oldManifestFile.flags.contains(EDepotFileFlag.Executable) + ) { + fileFinalPath.toFile().setExecutable(false) + } + + val fileStreamData = FileStreamData( + fileHandle = null, + fileLock = Mutex(), + chunksToDownload = AtomicInteger(neededChunks!!.size) + ) + + neededChunks!!.forEach { chunk -> + networkChunkQueue.send( + NetworkChunkItem( + fileStreamData = fileStreamData, + fileData = file, + chunk = chunk, + totalChunksForFile = neededChunks!!.size + ) + ) + } + } + + private suspend fun downloadSteam3DepotFileChunk( + downloadCounter: GlobalDownloadCounter, + depotFilesData: DepotFilesData, + file: FileData, + fileStreamData: FileStreamData, + chunk: ChunkData, + ): Unit = withContext(Dispatchers.IO) { + ensureActive() + + val depot = depotFilesData.depotDownloadInfo + val depotDownloadCounter = depotFilesData.depotCounter + + val chunkID = Strings.toHex(chunk.chunkID) + + var written = 0 + val chunkBuffer = ByteArray(chunk.uncompressedLength) + + try { + do { + ensureActive() + + var connection: Server? = null + + try { + connection = cdnClientPool!!.getConnection() + + var cdnToken: String? = null + + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") + } + } + + logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") + + written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( + depotId = depot.depotId, + chunk = chunk, + server = connection, + destination = chunkBuffer, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) + + cdnClientPool!!.returnConnection(connection) + + break + } catch (e: CancellationException) { + logger?.error(e) + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, + // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above + if (e.statusCode == 403 && + ( + !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host) || + steam3!!.cdnAuthTokens[depot.depotId to connection.host]?.isCompleted == false + ) + ) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + + cdnClientPool!!.returnConnection(connection) + + continue + } + + cdnClientPool!!.returnBrokenConnection(connection) + + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") + break + } + + logger?.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered unexpected error downloading chunk $chunkID", e) + } + } while (written == 0) + + if (written == 0) { + logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") + cancel() + } + + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() + + try { + fileStreamData.fileLock.lock() + + if (fileStreamData.fileHandle == null) { + val fileFinalPath = depot.installDir / file.fileName + fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) + } + + fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) + } finally { + fileStreamData.fileLock.unlock() + } + } finally { + } + + val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() + if (remainingChunks == 0) { + fileStreamData.fileHandle?.close() + } + + var sizeDownloaded = 0L + synchronized(depotDownloadCounter) { + sizeDownloaded = depotDownloadCounter.sizeDownloaded + written.toLong() + depotDownloadCounter.sizeDownloaded = sizeDownloaded + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + + val now = System.currentTimeMillis() + val fileKey = "${depot.depotId}:${file.fileName}" + val lastUpdate = lastFileProgressUpdate[fileKey] ?: 0L + + if (now - lastUpdate >= progressUpdateInterval || remainingChunks == 0) { + lastFileProgressUpdate[fileKey] = now + + val totalChunks = file.chunks.size + val completedChunks = totalChunks - remainingChunks + + // Approximate bytes based on completion ratio + val approximateBytesDownloaded = (file.totalSize * completedChunks) / totalChunks + + notifyListeners { listener -> + listener.onFileProgress( + depotId = depot.depotId, + fileName = file.fileName, + progress = FileProgress( + fileName = file.fileName, + bytesDownloaded = approximateBytesDownloaded, + totalBytes = file.totalSize, + chunksCompleted = completedChunks, + totalChunks = totalChunks, + status = if (remainingChunks == 0) DownloadStatus.COMPLETED else DownloadStatus.DOWNLOADING + ) + ) + } + } + + if (remainingChunks == 0) { + val fileFinalPath = depot.installDir / file.fileName + val percentage = (sizeDownloaded / depotDownloadCounter.completeDownloadSize.toFloat()) * 100.0f + logger?.debug("%.2f%% %s".format(percentage, fileFinalPath)) + } + } + + private fun testIsFileIncluded(filename: String): Boolean { + if (!config.usingFileList) { + return true + } + + val normalizedFilename = filename.replace('\\', '/') + + if (normalizedFilename in config.filesToDownload) { + return true + } + + for (regex in config.filesToDownloadRegex) { + if (regex.matches(normalizedFilename)) { + return true + } + } + + return false + } + + // endregion + + // region [REGION] Listener Operations + + fun addListener(listener: IDownloadListener) { + listeners.add(listener) + } + + fun removeListener(listener: IDownloadListener) { + listeners.remove(listener) + } + + private fun notifyListeners(action: (IDownloadListener) -> Unit) { + listeners.forEach { listener -> action(listener) } + } + + // endregion + + // region [REGION] Array Operations + + fun getItems(): List = items.toList() + + fun size(): Int = items.size + + fun isEmpty(): Boolean = items.isEmpty() + + fun get(index: Int): DownloadItem? = items.getOrNull(index) + + fun contains(item: DownloadItem): Boolean = items.contains(item) + + fun indexOf(item: DownloadItem): Int = items.indexOf(item) + + fun addAll(items: List) { + items.forEach(::add) + } + + fun add(item: DownloadItem) { + val index = items.size + items.add(item) + + if (isStarted.get()) { + remainingItems.incrementAndGet() + scope.launch { processingChannel.send(item) } + } + + notifyListeners { it.onItemAdded(item, index) } + } + + fun addFirst(item: DownloadItem) { + if (isStarted.get()) { + logger?.debug("Cannot add item when started.") + return + } + + items.add(0, item) + + notifyListeners { it.onItemAdded(item, 0) } + } + + fun addAt(index: Int, item: DownloadItem): Boolean { + if (isStarted.get()) { + logger?.debug("Cannot addAt item when started.") + return false + } + + return try { + items.add(index, item) + notifyListeners { it.onItemAdded(item, index) } + true + } catch (e: IndexOutOfBoundsException) { + false + } + } + + fun removeFirst(): DownloadItem? { + if (isStarted.get()) { + logger?.debug("Cannot removeFirst item when started.") + return null + } + + return if (items.isNotEmpty()) { + val item = items.removeAt(0) + notifyListeners { it.onItemRemoved(item, 0) } + item + } else { + null + } + } + + fun removeLast(): DownloadItem? { + if (isStarted.get()) { + logger?.debug("Cannot removeLast item when started.") + return null + } + + return if (items.isNotEmpty()) { + val lastIndex = items.size - 1 + val item = items.removeAt(lastIndex) + notifyListeners { it.onItemRemoved(item, lastIndex) } + item + } else { + null + } + } + + fun remove(item: DownloadItem): Boolean { + if (isStarted.get()) { + logger?.debug("Cannot remove item when started.") + return false + } + + val index = items.indexOf(item) + return if (index >= 0) { + items.removeAt(index) + notifyListeners { it.onItemRemoved(item, index) } + true + } else { + false + } + } + + fun removeAt(index: Int): DownloadItem? { + if (isStarted.get()) { + logger?.debug("Cannot removeAt item when started.") + return null + } + + return try { + val item = items.removeAt(index) + notifyListeners { it.onItemRemoved(item, index) } + item + } catch (e: IndexOutOfBoundsException) { + null + } + } + + fun moveItem(fromIndex: Int, toIndex: Int): Boolean { + if (isStarted.get()) { + logger?.debug("Cannot moveItem item when started.") + return false + } + + return try { + val item = items.removeAt(fromIndex) + items.add(toIndex, item) + true + } catch (e: IndexOutOfBoundsException) { + false + } + } + + fun clear() { + if (isStarted.get()) { + logger?.debug("Cannot clear item when started.") + return + } + + val oldItems = items.toList() + items.clear() + + notifyListeners { it.onQueueCleared(oldItems) } + } + + // endregion + + /** + * Some android emulators prefer using "Windows", so this will set downloading to prefer the Windows version. + */ + fun setAndroidEmulation(value: Boolean) { + if (isStarted.get()) { + logger?.error("Can't set android emulation value once started.") + return + } + + config = config.copy(androidEmulation = value) + + notifyListeners { it.onAndroidEmulation(config.androidEmulation) } + } + + @Throws(IllegalStateException::class) + fun start(): CompletableFuture = scope.future { + if (isStarted.getAndSet(true)) { + logger?.debug("Downloading already started.") + return@future false + } + + val initialItems = items.toList() + if (initialItems.isEmpty()) { + logger?.debug("No items to download") + return@future false + } + + // Send initial items + remainingItems.set(initialItems.size) + initialItems.forEach { processingChannel.send(it) } + + repeat(remainingItems.get()) { + // Process exactly this many + ensureActive() + + // Obtain the next item in queue. + val item = processingChannel.receive() + + try { + runBlocking { + if (useLanCache) { + ClientLancache.detectLancacheServer() + } + + if (ClientLancache.useLanCacheServer) { + logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") + + // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely + // be served much faster than over the internet. Steam internally has this behavior as well. + if (maxDownloads == 8) { + maxDownloads = 25 + } + } + + // Set some configuration values, first. + config = config.copy( + downloadManifestOnly = item.downloadManifestOnly, + installPath = item.installDirectory?.toPath(), + installToGameNameDirectory = item.installToGameNameDirectory, + ) + + // Sequential looping. + when (item) { + is PubFileItem -> { + if (item.pubfile == INVALID_MANIFEST_ID) { + logger?.debug("Invalid Pub File ID for ${item.appId}") + return@runBlocking + } + + logger?.debug("Downloading PUB File for ${item.appId}") + + notifyListeners { it.onDownloadStarted(item) } + downloadPubFile(item.appId, item.pubfile) + } + + is UgcItem -> { + if (item.ugcId == INVALID_MANIFEST_ID) { + logger?.debug("Invalid UGC ID for ${item.appId}") + return@runBlocking + } + + logger?.debug("Downloading UGC File for ${item.appId}") + + notifyListeners { it.onDownloadStarted(item) } + downloadUGC(item.appId, item.ugcId) + } + + is AppItem -> { + val branch = item.branch ?: DEFAULT_BRANCH + config = config.copy(betaPassword = item.branchPassword) + + if (!config.betaPassword.isNullOrBlank() && branch.isBlank()) { + logger?.error("Error: Cannot specify 'branchpassword' when 'branch' is not specified.") + return@runBlocking + } + + config = config.copy(downloadAllPlatforms = item.downloadAllPlatforms) + + val os = item.os + + if (config.downloadAllPlatforms && !os.isNullOrBlank()) { + logger?.error("Error: Cannot specify `os` when `all-platforms` is specified.") + return@runBlocking + } + + config = config.copy(downloadAllArchs = item.downloadAllArchs) + + val arch = item.osArch + + if (config.downloadAllArchs && !arch.isNullOrBlank()) { + logger?.error("Error: Cannot specify `osarch` when `all-archs` is specified.") + return@runBlocking + } + + config = config.copy(downloadAllLanguages = item.downloadAllLanguages) + + val language = item.language + + if (config.downloadAllLanguages && !language.isNullOrBlank()) { + logger?.error("Error: Cannot specify `language` when `all-languages` is specified.") + return@runBlocking + } + + val lv = item.lowViolence + + val depotManifestIds = mutableListOf>() + val isUGC = false + + val depotIdList = item.depot + val manifestIdList = item.manifest + + if (manifestIdList.isNotEmpty()) { + if (depotIdList.size != manifestIdList.size) { + logger?.error("Error: `manifest` requires one id for every `depot` specified") + return@runBlocking + } + val zippedDepotManifest = depotIdList.zip(manifestIdList) { depotId, manifestId -> + Pair(depotId, manifestId) + } + depotManifestIds.addAll(zippedDepotManifest) + } else { + depotManifestIds.addAll( + depotIdList.map { depotId -> + Pair(depotId, INVALID_MANIFEST_ID) + } + ) + } + + logger?.debug("Downloading App for ${item.appId}") + + notifyListeners { it.onDownloadStarted(item) } + downloadApp( + appId = item.appId, + depotManifestIds = depotManifestIds, + branch = branch, + os = os, + arch = arch, + language = language, + lv = lv, + isUgc = isUGC, + ) + } + } + + notifyListeners { it.onDownloadCompleted(item) } + } + } catch (e: IOException) { + logger?.error("Error downloading item ${item.appId}: ${e.message}", e) + + notifyListeners { it.onDownloadFailed(item, e) } + + throw e + } + } + + return@future true + } + + override fun close() { + isStarted.set(false) + + HttpClient.close() + + items.clear() + processingChannel.close() + + lastFileProgressUpdate.clear() + listeners.clear() + + steam3?.close() + steam3 = null + + cdnClientPool?.close() + cdnClientPool = null + + logger = null + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt new file mode 100644 index 00000000..5a514926 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt @@ -0,0 +1,7 @@ +package `in`.dragonbra.javasteam.depotdownloader + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class ContentDownloaderException(value: String) : Exception(value) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt new file mode 100644 index 00000000..2f746eb2 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -0,0 +1,53 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import okio.FileSystem +import okio.Path + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +@Serializable +data class DepotConfigStore( + val installedManifestIDs: MutableMap = mutableMapOf(), +) { + companion object { + private var instance: DepotConfigStore? = null + + private var filePath: Path? = null + + private val json = Json { prettyPrint = true } + + val isLoaded: Boolean + get() = instance != null + + fun loadFromFile(path: Path) { + // require(!isLoaded) { "Config already loaded" } + + instance = if (FileSystem.SYSTEM.exists(path)) { + FileSystem.SYSTEM.read(path) { + json.decodeFromString(readUtf8()) + } + } else { + DepotConfigStore() + } + + filePath = path + } + + fun save() { + val currentInstance = requireNotNull(instance) { "Saved config before loading" } + val currentPath = requireNotNull(filePath) { "File path not set" } + + currentPath.parent?.let { FileSystem.SYSTEM.createDirectories(it) } + + FileSystem.SYSTEM.write(currentPath) { + writeUtf8(json.encodeToString(currentInstance)) + } + } + + fun getInstance(): DepotConfigStore = requireNotNull(instance) { "Config not loaded" } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt new file mode 100644 index 00000000..03908dc2 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -0,0 +1,46 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.util.Versions +import io.ktor.client.HttpClient +import io.ktor.client.engine.cio.CIO +import io.ktor.client.engine.cio.endpoint +import io.ktor.client.plugins.UserAgent +import kotlinx.coroutines.isActive + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +object HttpClient { + + private var _httpClient: HttpClient? = null + + val httpClient: HttpClient + get() { + if (_httpClient?.isActive != true) { + _httpClient = HttpClient(CIO) { + install(UserAgent) { + agent = "DepotDownloader/${Versions.getVersion()}" + } + engine { + maxConnectionsCount = 10 + endpoint { + maxConnectionsPerRoute = 5 + pipelineMaxSize = 20 + keepAliveTime = 5000 + connectTimeout = 5000 + requestTimeout = 30000 + } + } + } + } + return _httpClient!! + } + + fun close() { + if (httpClient.isActive) { + _httpClient?.close() + _httpClient = null + } + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt new file mode 100644 index 00000000..c3ce3c22 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -0,0 +1,35 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem +import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress +import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress + +/** + * Listener interface for download events. + * + * @author Lossy + * @since Oct 1, 2025 + */ +interface IDownloadListener { + // Queue management + fun onItemAdded(item: DownloadItem, index: Int) {} + fun onItemRemoved(item: DownloadItem, index: Int) {} + fun onQueueCleared(previousItems: List) {} + + // Download lifecycle + fun onDownloadStarted(item: DownloadItem) {} + fun onDownloadCompleted(item: DownloadItem) {} + fun onDownloadFailed(item: DownloadItem, error: Throwable) {} + + // Progress tracking + fun onOverallProgress(progress: OverallProgress) {} + fun onDepotProgress(depotId: Int, progress: DepotProgress) {} + fun onFileProgress(depotId: Int, fileName: String, progress: FileProgress) {} + + // Status updates + fun onStatusUpdate(message: String) {} + + // Configuration + fun onAndroidEmulation(value: Boolean) {} +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt new file mode 100644 index 00000000..2b80f563 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -0,0 +1,340 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.enums.EResult +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesPublishedfileSteamclient +import `in`.dragonbra.javasteam.rpc.service.PublishedFile +import `in`.dragonbra.javasteam.steam.cdn.Server +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo +import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest +import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.SteamCloud +import `in`.dragonbra.javasteam.steam.handlers.steamcloud.callback.UGCDetailsCallback +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.CDNAuthToken +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent +import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages +import `in`.dragonbra.javasteam.steam.handlers.steamuser.SteamUser +import `in`.dragonbra.javasteam.steam.steamclient.SteamClient +import `in`.dragonbra.javasteam.types.KeyValue +import `in`.dragonbra.javasteam.types.PublishedFileID +import `in`.dragonbra.javasteam.types.UGCHandle +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import java.io.Closeable +import java.util.concurrent.ConcurrentHashMap + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class Steam3Session( + private val steamClient: SteamClient, + debug: Boolean, +) : Closeable { + + private var logger: Logger? = null + + internal val appTokens = ConcurrentHashMap() + internal val packageTokens = ConcurrentHashMap() + internal val depotKeys = ConcurrentHashMap() + internal val cdnAuthTokens = ConcurrentHashMap, CompletableDeferred>() + internal val appInfo = ConcurrentHashMap() + internal val packageInfo = ConcurrentHashMap() + internal val appBetaPasswords = ConcurrentHashMap() + + private var unifiedMessages: SteamUnifiedMessages? = null + internal var steamUser: SteamUser? = null + internal var steamContent: SteamContent? = null + internal var steamApps: SteamApps? = null + internal var steamCloud: SteamCloud? = null + internal var steamPublishedFile: PublishedFile? = null + + init { + if (debug) { + logger = LogManager.getLogger(Steam3Session::class.java) + } + + unifiedMessages = requireNotNull(steamClient.getHandler()) + steamApps = requireNotNull(steamClient.getHandler()) + steamCloud = requireNotNull(steamClient.getHandler()) + steamContent = requireNotNull(steamClient.getHandler()) + steamPublishedFile = requireNotNull(unifiedMessages?.createService()) + steamUser = requireNotNull(steamClient.getHandler()) + } + + override fun close() { + logger?.debug("Closing...") + + unifiedMessages = null + steamUser = null + steamContent = null + steamApps = null + steamCloud = null + steamPublishedFile = null + + cdnAuthTokens.values.forEach { it.cancel() } + cdnAuthTokens.clear() + + depotKeys.values.forEach { it.fill(0) } + depotKeys.clear() + appBetaPasswords.values.forEach { it.fill(0) } + appBetaPasswords.clear() + + appTokens.clear() + packageTokens.clear() + appInfo.clear() + packageInfo.clear() + + logger = null + } + + suspend fun requestAppInfo(appId: Int, bForce: Boolean = false) { + if (appInfo.containsKey(appId) && !bForce) { + return + } + + val appTokens = steamApps!!.picsGetAccessTokens(appId).await() + + if (appTokens.appTokensDenied.contains(appId)) { + logger?.error("Insufficient privileges to get access token for app $appId") + } + + appTokens.appTokens.forEach { tokenDict -> + this.appTokens[tokenDict.key] = tokenDict.value + } + + val request = PICSRequest(appId) + + this.appTokens[appId]?.let { token -> + request.accessToken = token + } + + val appInfoMultiple = steamApps!!.picsGetProductInfo(request).await() + + logger?.debug( + "requestAppInfo($appId, $bForce) with \n" + + "${appTokens.appTokens.size} appTokens, \n" + + "${appTokens.appTokensDenied.size} appTokensDenied, \n" + + "${appTokens.packageTokens.size} packageTokens, and \n" + + "${appTokens.packageTokensDenied} packageTokensDenied. \n" + + "picsGetProductInfo result size: ${appInfoMultiple.results.size}" + ) + + appInfoMultiple.results.forEach { appInfo -> + appInfo.apps.forEach { appValue -> + val app = appValue.value + this.appInfo[app.id] = app + } + appInfo.unknownApps.forEach { app -> + this.appInfo[app] = null + } + } + } + + // TODO race condition (??) + private val packageInfoMutex = Mutex() + suspend fun requestPackageInfo(packageIds: List) { + packageInfoMutex.withLock { + // I have a silly race condition??? + val packages = packageIds.filter { !packageInfo.containsKey(it) } + + if (packages.isEmpty()) return + + val packageRequests = arrayListOf() + + packages.forEach { pkg -> + val request = PICSRequest(id = pkg) + + packageTokens[pkg]?.let { token -> + request.accessToken = token + } + + packageRequests.add(request) + } + + val packageInfoMultiple = steamApps!!.picsGetProductInfo(emptyList(), packageRequests).await() + + logger?.debug( + "requestPackageInfo(packageIds =${packageIds.size}) \n" + + "picsGetProductInfo result size: ${packageInfoMultiple.results.size} " + ) + + packageInfoMultiple.results.forEach { pkgInfo -> + pkgInfo.packages.forEach { pkgValue -> + val pkg = pkgValue.value + packageInfo[pkg.id] = pkg + } + pkgInfo.unknownPackages.forEach { pkgValue -> + packageInfo[pkgValue] = null + } + } + } + } + + suspend fun requestFreeAppLicense(appId: Int): Boolean { + try { + val resultInfo = steamApps!!.requestFreeLicense(appId).await() + + logger?.debug("requestFreeAppLicense($appId) has result ${resultInfo.result}") + + return resultInfo.grantedApps.contains(appId) + } catch (e: Exception) { + logger?.error("Failed to request FreeOnDemand license for app $appId: ${e.message}") + return false + } + } + + suspend fun requestDepotKey(depotId: Int, appId: Int = 0) { + if (depotKeys.containsKey(depotId)) { + return + } + + val depotKey = steamApps!!.getDepotDecryptionKey(depotId, appId).await() + + logger?.debug( + "requestDepotKey($depotId, $appId) " + + "Got depot key for ${depotKey.depotID} result: ${depotKey.result}" + ) + + if (depotKey.result != EResult.OK) { + return + } + + depotKeys[depotKey.depotID] = depotKey.depotKey + } + + suspend fun getDepotManifestRequestCode( + depotId: Int, + appId: Int, + manifestId: Long, + branch: String, + ): ULong = withContext(Dispatchers.IO) { + val requestCode = steamContent!!.getManifestRequestCode( + depotId = depotId, + appId = appId, + manifestId = manifestId, + branch = branch, + branchPasswordHash = null, + parentScope = this // TODO am I passing this right? + ).await().toULong() + + if (requestCode == 0UL) { + logger?.error("No manifest request code was returned for depot $depotId from app $appId, manifest $manifestId") + + if (steamClient.isDisconnected) { + logger?.debug("Suggestion: Try logging in with -username as old manifests may not be available for anonymous accounts.") + } + } else { + logger?.debug("Got manifest request code for depot $depotId from app $appId, manifest $manifestId, result: $requestCode") + } + + logger?.debug( + "getDepotManifestRequestCode($depotId, $appId, $manifestId, $branch) " + + "got request code $requestCode" + ) + + return@withContext requestCode + } + + suspend fun requestCDNAuthToken(appId: Int, depotId: Int, server: Server) = withContext(Dispatchers.IO) { + val cdnKey = depotId to server.host!! + + if (cdnAuthTokens.containsKey(cdnKey)) { + return@withContext + } + + val completion = CompletableDeferred() + + val existing = cdnAuthTokens.putIfAbsent(cdnKey, completion) + if (existing != null) { + return@withContext + } + + logger?.debug("Requesting CDN auth token for ${server.host}") + + try { + val cdnAuth = steamContent!!.getCDNAuthToken(appId, depotId, server.host!!, this).await() + + logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") + + if (cdnAuth.result != EResult.OK) { + cdnAuthTokens.remove(cdnKey) // Remove failed promise + completion.completeExceptionally(Exception("Failed to get CDN auth token: ${cdnAuth.result}")) + return@withContext + } + + completion.complete(cdnAuth) + } catch (e: Exception) { + logger?.error(e) + cdnAuthTokens.remove(cdnKey) // Remove failed promise + completion.completeExceptionally(e) + } + } + + suspend fun checkAppBetaPassword(appId: Int, password: String) { + val appPassword = steamApps!!.checkAppBetaPassword(appId, password).await() + + logger?.debug( + "checkAppBetaPassword($appId, )," + + "retrieved ${appPassword.betaPasswords.size} beta keys with result: ${appPassword.result}" + ) + + appPassword.betaPasswords.forEach { entry -> + this.appBetaPasswords[entry.key] = entry.value + } + } + + suspend fun getPrivateBetaDepotSection(appId: Int, branch: String): KeyValue { + // Should be filled by CheckAppBetaPassword + val branchPassword = appBetaPasswords[branch] ?: return KeyValue() + + // Should be filled by RequestAppInfo + val accessToken = appTokens[appId] ?: 0L + + val privateBeta = steamApps!!.picsGetPrivateBeta(appId, accessToken, branch, branchPassword).await() + + logger?.debug("getPrivateBetaDepotSection($appId, $branch) result: ${privateBeta.result}") + + return privateBeta.depotSection + } + + @Throws(ContentDownloaderException::class) + suspend fun getPublishedFileDetails( + appId: Int, + pubFile: PublishedFileID, + ): SteammessagesPublishedfileSteamclient.PublishedFileDetails? { + val pubFileRequest = + SteammessagesPublishedfileSteamclient.CPublishedFile_GetDetails_Request.newBuilder().apply { + this.appid = appId + this.addPublishedfileids(pubFile.toLong()) + }.build() + + val details = steamPublishedFile!!.getDetails(pubFileRequest).await() + + logger?.debug("requestUGCDetails($appId, $pubFile) result: ${details.result}") + + if (details.result == EResult.OK) { + return details.body.publishedfiledetailsBuilderList.firstOrNull()?.build() + } + + throw ContentDownloaderException("EResult ${details.result.code()} (${details.result}) while retrieving file details for pubfile $pubFile.") + } + + suspend fun getUGCDetails(ugcHandle: UGCHandle): UGCDetailsCallback? { + val callback = steamCloud!!.requestUGCDetails(ugcHandle).await() + + logger?.debug("requestUGCDetails($ugcHandle) result: ${callback.result}") + + if (callback.result == EResult.OK) { + return callback + } else if (callback.result == EResult.FileNotFound) { + return null + } + + throw ContentDownloaderException("EResult ${callback.result.code()} (${callback.result}) while retrieving UGC details for ${ugcHandle.value}.") + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt new file mode 100644 index 00000000..b69a0047 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -0,0 +1,207 @@ +package `in`.dragonbra.javasteam.depotdownloader + +import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo +import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.types.ChunkData +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.util.Adler32 +import okio.FileHandle +import okio.FileSystem +import okio.Path +import okio.Path.Companion.toPath +import okio.buffer +import org.apache.commons.lang3.SystemUtils +import java.io.IOException +import java.security.MessageDigest + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +object Util { + + @JvmOverloads + @JvmStatic + fun getSteamOS(androidEmulation: Boolean = false): String { + if (SystemUtils.IS_OS_WINDOWS) { + return "windows" + } + if (SystemUtils.IS_OS_MAC_OSX) { + return "macos" + } + if (SystemUtils.IS_OS_LINUX) { + return "linux" + } + if (SystemUtils.IS_OS_FREE_BSD) { + // Return linux as freebsd steam client doesn't exist yet + return "linux" + } + + // Hack for PC emulation on android. (Pluvia, GameNative, GameHub) + if (androidEmulation && SystemUtils.IS_OS_ANDROID) { + return "windows" + } + + return "unknown" + } + + @JvmStatic + fun getSteamArch(): String { + val arch = System.getProperty("os.arch")?.lowercase() ?: "" + return when { + arch.contains("64") -> "64" + arch.contains("86") -> "32" + arch.contains("amd64") -> "64" + arch.contains("x86_64") -> "64" + arch.contains("aarch64") -> "64" + arch.contains("arm64") -> "64" + else -> "32" + } + } + + @JvmStatic + fun saveManifestToFile(directory: Path, manifest: DepotManifest): Boolean = try { + val filename = directory / "${manifest.depotID}_${manifest.manifestGID}.manifest" + manifest.saveToFile(filename.toString()) + + val shaFile = "$filename.sha".toPath() + FileSystem.SYSTEM.write(shaFile) { + write(fileSHAHash(filename)) + } + + true + } catch (e: Exception) { + false + } + + @JvmStatic + fun loadManifestFromFile( + directory: Path, + depotId: Int, + manifestId: Long, + badHashWarning: Boolean, + ): DepotManifest? { + // Try loading Steam format manifest first. + val filename = directory / "${depotId}_$manifestId.manifest" + + if (FileSystem.SYSTEM.exists(filename)) { + val expectedChecksum = try { + FileSystem.SYSTEM.read(filename / ".sha") { + readByteArray() + } + } catch (e: IOException) { + null + } + + val currentChecksum = fileSHAHash(filename) + + if (expectedChecksum != null && expectedChecksum.contentEquals(currentChecksum)) { + return DepotManifest.loadFromFile(filename.toString()) + } else if (badHashWarning) { + println("Manifest $manifestId on disk did not match the expected checksum.") + } + } + + return null + } + + @JvmStatic + fun fileSHAHash(filename: Path): ByteArray { + val digest = MessageDigest.getInstance("SHA-1") + + FileSystem.SYSTEM.source(filename).use { source -> + source.buffer().use { bufferedSource -> + val buffer = ByteArray(8192) + var bytesRead: Int + + while (bufferedSource.read(buffer).also { bytesRead = it } != -1) { + digest.update(buffer, 0, bytesRead) + } + } + } + + return digest.digest() + } + + /** + * Validate a file against Steam3 Chunk data + * + * @param handle FileHandle to read from + * @param chunkData Array of ChunkData to validate against + * @return List of ChunkData that are needed + * @throws IOException If there's an error reading the file + */ + @Throws(IOException::class) + fun validateSteam3FileChecksums(handle: FileHandle, chunkData: List): List { + val neededChunks = mutableListOf() + + for (data in chunkData) { + val chunk = ByteArray(data.uncompressedLength) + val read = handle.read(data.offset, chunk, 0, data.uncompressedLength) + + val tempChunk = if (read > 0 && read < data.uncompressedLength) { + chunk.copyOf(read) + } else { + chunk + } + + val adler = Adler32.calculate(tempChunk) + if (adler != data.checksum) { + neededChunks.add(data) + } + } + + return neededChunks + } + + @JvmStatic + fun dumpManifestToTextFile(depot: DepotDownloadInfo, manifest: DepotManifest) { + val txtManifest = depot.installDir / "manifest_${depot.depotId}_${depot.manifestId}.txt" + + FileSystem.SYSTEM.write(txtManifest) { + writeUtf8("Content Manifest for Depot ${depot.depotId}\n") + writeUtf8("\n") + writeUtf8("Manifest ID / date : ${depot.manifestId} / ${manifest.creationTime}\n") + + val uniqueChunks = manifest.files + .flatMap { it.chunks } + .mapNotNull { it.chunkID } + .toSet() + + writeUtf8("Total number of files : ${manifest.files.size}\n") + writeUtf8("Total number of chunks : ${uniqueChunks.size}\n") + writeUtf8("Total bytes on disk : ${manifest.totalUncompressedSize}\n") + writeUtf8("Total bytes compressed : ${manifest.totalCompressedSize}\n") + writeUtf8("\n") + writeUtf8("\n") + + writeUtf8(" Size Chunks File SHA Flags Name\n") + manifest.files.forEach { file -> + val sha1Hash = file.fileHash.toHexString().lowercase() + writeUtf8( + "%14d %6d %s %5x %s\n".format( + file.totalSize, + file.chunks.size, + sha1Hash, + EDepotFileFlag.code(file.flags), + file.fileName + ) + ) + } + } + } + + @JvmStatic + fun formatBytes(bytes: Long): String { + val units = arrayOf("B", "KB", "MB", "GB") + var size = bytes.toDouble() + var unitIndex = 0 + + while (size >= 1024 && unitIndex < units.size - 1) { + size /= 1024 + unitIndex++ + } + + return "%.2f %s".format(size, units[unitIndex]) + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt new file mode 100644 index 00000000..05fbb247 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt @@ -0,0 +1,9 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.types.ChunkData + +/** + * @author Oxters + * @since Oct 29, 2024 + */ +data class ChunkMatch(val oldChunk: ChunkData, val newChunk: ChunkData) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt new file mode 100644 index 00000000..256575ec --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt @@ -0,0 +1,41 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import okio.Path + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotDownloadInfo( + val depotId: Int, + val appId: Int, + val manifestId: Long, + val branch: String, + val installDir: Path, + val depotKey: ByteArray, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is DepotDownloadInfo) return false + + if (depotId != other.depotId) return false + if (appId != other.appId) return false + if (manifestId != other.manifestId) return false + if (branch != other.branch) return false + if (installDir != other.installDir) return false + if (!depotKey.contentEquals(other.depotKey)) return false + + return true + } + + override fun hashCode(): Int { + var result = depotId + result = 31 * result + appId + result = 31 * result + manifestId.hashCode() + result = 31 * result + branch.hashCode() + result = 31 * result + installDir.hashCode() + result = 31 * result + depotKey.contentHashCode() + return result + } +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt new file mode 100644 index 00000000..bef4590b --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt @@ -0,0 +1,20 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.types.DepotManifest +import `in`.dragonbra.javasteam.types.FileData +import okio.Path + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotFilesData( + val depotDownloadInfo: DepotDownloadInfo, + val depotCounter: DepotDownloadCounter, + val stagingDir: Path, + val manifest: DepotManifest, + val previousManifest: DepotManifest?, + val filteredFiles: MutableList, + val allFileNames: HashSet, +) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt new file mode 100644 index 00000000..8d509e27 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt @@ -0,0 +1,26 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +// https://kotlinlang.org/docs/coding-conventions.html#source-file-organization + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class GlobalDownloadCounter( + var completeDownloadSize: Long = 0, + var totalBytesCompressed: Long = 0, + var totalBytesUncompressed: Long = 0, +) + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class DepotDownloadCounter( + var completeDownloadSize: Long = 0, + var sizeDownloaded: Long = 0, + var depotBytesCompressed: Long = 0, + var depotBytesUncompressed: Long = 0, +) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt new file mode 100644 index 00000000..a204c4e1 --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -0,0 +1,62 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import `in`.dragonbra.javasteam.depotdownloader.ContentDownloader + +// https://kotlinlang.org/docs/coding-conventions.html#source-file-organization + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +abstract class DownloadItem( + val appId: Int, + val installDirectory: String?, + val installToGameNameDirectory: Boolean, + val downloadManifestOnly: Boolean, +) + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class UgcItem @JvmOverloads constructor( + appId: Int, + val ugcId: Long = ContentDownloader.INVALID_MANIFEST_ID, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class PubFileItem @JvmOverloads constructor( + appId: Int, + val pubfile: Long = ContentDownloader.INVALID_MANIFEST_ID, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) + +/** + * @author Lossy + * @since Oct 1, 2025 + */ +class AppItem @JvmOverloads constructor( + appId: Int, + installToGameNameDirectory: Boolean = false, + installDirectory: String? = null, + val branch: String? = null, + val branchPassword: String? = null, + val downloadAllPlatforms: Boolean = false, + val os: String? = null, + val downloadAllArchs: Boolean = false, + val osArch: String? = null, + val downloadAllLanguages: Boolean = false, + val language: String? = null, + val lowViolence: Boolean = false, + val depot: List = emptyList(), + val manifest: List = emptyList(), + downloadManifestOnly: Boolean = false, +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt new file mode 100644 index 00000000..f8e8035d --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt @@ -0,0 +1,72 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +/** + * Overall download progress across all items. + * + * @author Lossy + * @since Oct 1, 2025 + */ +data class OverallProgress( + val currentItem: Int, + val totalItems: Int, + val totalBytesDownloaded: Long, + val totalBytesExpected: Long, + val status: DownloadStatus, +) { + val percentComplete: Double + get() = if (totalBytesExpected > 0) { + (totalBytesDownloaded.toDouble() / totalBytesExpected) * 100.0 + } else { + 0.0 + } +} + +/** + * Progress for a specific depot + * + * @author Lossy + * @since Oct 1, 2025 + */ +data class DepotProgress( + val depotId: Int, + val filesCompleted: Int, + val totalFiles: Int, + val bytesDownloaded: Long, + val totalBytes: Long, + val status: DownloadStatus, +) { + val percentComplete: Double + get() = if (totalBytes > 0) { + (bytesDownloaded.toDouble() / totalBytes) * 100.0 + } else { + 0.0 + } +} + +/** + * Progress for a specific file + * + * @author Lossy + * @since Oct 1, 2025 + */ +data class FileProgress( + val fileName: String, + val bytesDownloaded: Long, + val totalBytes: Long, + val chunksCompleted: Int, + val totalChunks: Int, + val status: DownloadStatus, +) { + val percentComplete: Double + get() = if (totalBytes > 0) { + (bytesDownloaded.toDouble() / totalBytes) * 100.0 + } else { + 0.0 + } +} + +enum class DownloadStatus { + PREPARING, + DOWNLOADING, + COMPLETED, +} diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt new file mode 100644 index 00000000..2b6a539c --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt @@ -0,0 +1,16 @@ +package `in`.dragonbra.javasteam.depotdownloader.data + +import kotlinx.coroutines.sync.Mutex +import okio.FileHandle +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author Oxters + * @author Lossy + * @since Oct 29, 2024 + */ +data class FileStreamData( + var fileHandle: FileHandle?, + val fileLock: Mutex = Mutex(), + var chunksToDownload: AtomicInteger = AtomicInteger(0), +) diff --git a/javasteam-samples/build.gradle.kts b/javasteam-samples/build.gradle.kts index 9deffead..1e1aa774 100644 --- a/javasteam-samples/build.gradle.kts +++ b/javasteam-samples/build.gradle.kts @@ -14,12 +14,13 @@ java { dependencies { implementation(rootProject) implementation(project(":javasteam-cs")) + implementation(project(":javasteam-depotdownloader")) implementation(libs.bouncyCastle) implementation(libs.gson) implementation(libs.kotlin.coroutines) implementation(libs.okHttp) - implementation(libs.protobuf.java) // To access protobufs directly as shown in Sample #2 + implementation(libs.protobuf.java) // Protobuf access implementation(libs.qrCode) implementation(libs.zstd) // Content Downloading. implementation(libs.xz) // Content Downloading. diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index e22d7e08..b73f408a 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -1,9 +1,17 @@ package in.dragonbra.javasteamsamples._023_downloadapp; +import in.dragonbra.javasteam.depotdownloader.ContentDownloader; +import in.dragonbra.javasteam.depotdownloader.IDownloadListener; +import in.dragonbra.javasteam.depotdownloader.data.*; import in.dragonbra.javasteam.enums.EResult; -import in.dragonbra.javasteam.steam.contentdownloader.ContentDownloader; +import in.dragonbra.javasteam.steam.authentication.AuthPollResult; +import in.dragonbra.javasteam.steam.authentication.AuthSessionDetails; +import in.dragonbra.javasteam.steam.authentication.AuthenticationException; +import in.dragonbra.javasteam.steam.authentication.UserConsoleAuthenticator; +import in.dragonbra.javasteam.steam.handlers.steamapps.License; import in.dragonbra.javasteam.steam.handlers.steamapps.SteamApps; import in.dragonbra.javasteam.steam.handlers.steamapps.callback.FreeLicenseCallback; +import in.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback; import in.dragonbra.javasteam.steam.handlers.steamuser.LogOnDetails; import in.dragonbra.javasteam.steam.handlers.steamuser.SteamUser; import in.dragonbra.javasteam.steam.handlers.steamuser.callback.LoggedOffCallback; @@ -14,8 +22,17 @@ import in.dragonbra.javasteam.steam.steamclient.callbacks.DisconnectedCallback; import in.dragonbra.javasteam.util.log.DefaultLogListener; import in.dragonbra.javasteam.util.log.LogManager; +import org.jetbrains.annotations.NotNull; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; -import java.io.File; /** * @author Oxters @@ -32,15 +49,18 @@ * called Rocky Mayhem */ @SuppressWarnings("FieldCanBeLocal") -public class SampleDownloadApp implements Runnable { +public class SampleDownloadApp implements Runnable, IDownloadListener { private final int ROCKY_MAYHEM_APP_ID = 1303350; - private final int ROCKY_MAYHEM_DEPOT_ID = 1303351; + + private final String DEFAULT_INSTALL_DIRECTORY = "steamapps"; private SteamClient steamClient; private CallbackManager manager; + private SteamUser steamUser; + private SteamApps steamApps; private boolean isRunning; @@ -49,12 +69,13 @@ public class SampleDownloadApp implements Runnable { private final String pass; - private final String twoFactor; + private List subscriptions; + + private List licenseList; - public SampleDownloadApp(String user, String pass, String twoFactor) { + public SampleDownloadApp(String user, String pass) { this.user = user; this.pass = pass; - this.twoFactor = twoFactor; } public static void main(String[] args) { @@ -65,68 +86,111 @@ public static void main(String[] args) { LogManager.addListener(new DefaultLogListener()); - String twoFactor = null; - if (args.length == 3) - twoFactor = args[2]; - new SampleDownloadApp(args[0], args[1], twoFactor).run(); + new SampleDownloadApp(args[0], args[1]).run(); } @Override public void run() { - // create our steamclient instance + // Most everything has been described in earlier samples. + // Anything pertaining to this sample will be commented. + steamClient = new SteamClient(); - // create the callback manager which will route callbacks to function calls manager = new CallbackManager(steamClient); - // get the steamuser handler, which is used for logging on after successfully connecting steamUser = steamClient.getHandler(SteamUser.class); - steamApps = steamClient.getHandler(SteamApps.class); - // register a few callbacks we're interested in - // these are registered upon creation to a callback manager, which will then route the callbacks - // to the functions specified - manager.subscribe(ConnectedCallback.class, this::onConnected); - manager.subscribe(DisconnectedCallback.class, this::onDisconnected); + steamApps = steamClient.getHandler(SteamApps.class); - manager.subscribe(LoggedOnCallback.class, this::onLoggedOn); - manager.subscribe(LoggedOffCallback.class, this::onLoggedOff); + subscriptions = new ArrayList<>(); - manager.subscribe(FreeLicenseCallback.class, this::onFreeLicense); + subscriptions.add(manager.subscribe(ConnectedCallback.class, this::onConnected)); + subscriptions.add(manager.subscribe(DisconnectedCallback.class, this::onDisconnected)); + subscriptions.add(manager.subscribe(LoggedOnCallback.class, this::onLoggedOn)); + subscriptions.add(manager.subscribe(LoggedOffCallback.class, this::onLoggedOff)); + subscriptions.add(manager.subscribe(LicenseListCallback.class, this::onLicenseList)); + subscriptions.add(manager.subscribe(FreeLicenseCallback.class, this::onFreeLicense)); isRunning = true; System.out.println("Connecting to steam..."); - // initiate the connection steamClient.connect(); - // create our callback handling loop while (isRunning) { - // in order for the callbacks to get routed, they need to be handled by the manager manager.runWaitCallbacks(1000L); } + + for (var subscription : subscriptions) { + try { + subscription.close(); + } catch (IOException e) { + System.out.println("Couldn't close a callback."); + } + } } private void onConnected(ConnectedCallback callback) { System.out.println("Connected to Steam! Logging in " + user + "..."); - LogOnDetails details = new LogOnDetails(); - details.setUsername(user); - details.setPassword(pass); - if (twoFactor != null) { - details.setTwoFactorCode(twoFactor); - } + AuthSessionDetails authDetails = new AuthSessionDetails(); + authDetails.username = user; + authDetails.password = pass; + authDetails.deviceFriendlyName = "JavaSteam - Sample 023"; + authDetails.persistentSession = true; + + authDetails.authenticator = new UserConsoleAuthenticator(); + + try { + var path = Paths.get("refreshtoken.txt"); + + String accountName; + String refreshToken; + if (!Files.exists(path)) { + System.out.println("No existing refresh token found. Beginning Authentication"); + + var authSession = steamClient.getAuthentication().beginAuthSessionViaCredentials(authDetails).get(); + + AuthPollResult pollResponse = authSession.pollingWaitForResult().get(); - // Set LoginID to a non-zero value if you have another client connected using the same account, - // the same private ip, and same public ip. - details.setLoginID(149); + accountName = pollResponse.getAccountName(); + refreshToken = pollResponse.getRefreshToken(); - steamUser.logOn(details); + // Save out refresh token for automatic login on next sample run. + Files.writeString(path, pollResponse.getRefreshToken()); + } else { + System.out.println("Existing refresh token found"); + var token = Files.readString(path); + + accountName = user; + refreshToken = token; + } + + LogOnDetails details = new LogOnDetails(); + details.setUsername(accountName); + details.setAccessToken(refreshToken); + details.setShouldRememberPassword(true); + + details.setLoginID(149); + + System.out.println("Logging in..."); + + steamUser.logOn(details); + } catch (Exception e) { + if (e instanceof AuthenticationException) { + System.err.println("An Authentication error has occurred. " + e.getMessage()); + } else if (e instanceof CancellationException) { + System.err.println("An Cancellation exception was raised. Usually means a timeout occurred. " + e.getMessage()); + } else { + System.err.println("An error occurred:" + e.getMessage()); + } + + steamUser.logOff(); + } } private void onDisconnected(DisconnectedCallback callback) { - System.out.println("Disconnected from Steam"); + System.out.println("Disconnected from Steam, UserInitiated: " + callback.isUserInitiated()); if (callback.isUserInitiated()) { isRunning = false; @@ -143,9 +207,6 @@ private void onDisconnected(DisconnectedCallback callback) { private void onLoggedOn(LoggedOnCallback callback) { if (callback.getResult() != EResult.OK) { if (callback.getResult() == EResult.AccountLogonDenied) { - // if we receive AccountLogonDenied or one of its flavors (AccountLogonDeniedNoMailSent, etc.) - // then the account we're logging into is SteamGuard protected - // see sample 5 for how SteamGuard can be handled System.out.println("Unable to logon to Steam: This account is SteamGuard protected."); isRunning = false; @@ -163,9 +224,21 @@ private void onLoggedOn(LoggedOnCallback callback) { // now that we are logged in, we can request a free license for Rocky Mayhem steamApps.requestFreeLicense(ROCKY_MAYHEM_APP_ID); + } + private void onLicenseList(LicenseListCallback callback) { + if (callback.getResult() != EResult.OK) { + System.out.println("Failed to obtain licenses the account owns."); + steamClient.disconnect(); + return; + } + + licenseList = callback.getLicenseList(); + + System.out.println("Got " + licenseList.size() + " licenses from account!"); } + @SuppressWarnings("ExtractMethodRecommender") private void onFreeLicense(FreeLicenseCallback callback) { if (callback.getResult() != EResult.OK) { System.out.println("Failed to get a free license for Rocky Mayhem"); @@ -173,24 +246,72 @@ private void onFreeLicense(FreeLicenseCallback callback) { return; } - // we have successfully received a free license for Rocky Mayhem so now we can start the download process - // note: it is okay to see some errors about ContentDownloader failing to download a chunk, it will retry and continue. - new File("steamapps/staging/").mkdirs(); - var contentDownloader = new ContentDownloader(steamClient); - contentDownloader.downloadApp( - ROCKY_MAYHEM_APP_ID, - ROCKY_MAYHEM_DEPOT_ID, - "steamapps/", - "steamapps/staging/", - "public", - 8, - progress -> System.out.println("Download progress: " + progress) - ).thenAccept(success -> { + // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. + // You will need to subscribe to LicenseListCallback to obtain your app licenses. + try (var depotDownloader = new ContentDownloader(steamClient, licenseList, false)) { + + depotDownloader.addListener(this); + + // An app id is required at minimum for all item types. + var pubItem = new PubFileItem( + /* appId */ 0, + /* pubfile */ 0, + /* installToGameNameDirectory */ false, + /* installDirectory */ null, + /* downloadManifestOnly */ false + ); // TODO find actual pub item + + var ugcItem = new UgcItem( + /* appId */0, + /* ugcId */ 0, + /* installToGameNameDirectory */ false, + /* installDirectory */ null, + /* downloadManifestOnly */ false + ); // TODO find actual ugc item + + var appItem = new AppItem( + /* appId */ 204360, + /* installToGameNameDirectory */ true, + /* installDirectory */ DEFAULT_INSTALL_DIRECTORY, + /* branch */ "public", + /* branchPassword */ "", + /* downloadAllPlatforms */ false, + /* os */ "windows", + /* downloadAllArchs */ false, + /* osArch */ "64", + /* downloadAllLanguages */ false, + /* language */ "english", + /* lowViolence */ false, + /* depot */ List.of(), + /* manifest */ List.of(), + /* downloadManifestOnly */ false + ); + + var appItem2 = new AppItem(225840, true); + var appItem3 = new AppItem(3527290, true); + var appItem4 = new AppItem(ROCKY_MAYHEM_APP_ID, true); + + var downloadList = List.of(pubItem, ugcItem, appItem); + + // Add specified games to the queue. Add, Remove, Move, and general array manipulation methods are available. + // depotDownloader.addAll(downloadList); // TODO + depotDownloader.addAll(List.of(appItem)); + + // Start downloading your items. Array manipulation is now disabled. You can still add to the list. + var success = depotDownloader.start().get(); // Future + if (success) { System.out.println("Download completed successfully"); } - steamClient.disconnect(); - }); + + depotDownloader.removeListener(this); + } catch (IllegalStateException | InterruptedException | ExecutionException e) { + System.out.println("Something happened"); + System.err.println(e.getMessage()); + } finally { + steamUser.logOff(); + System.out.println("Done Downloading"); + } } private void onLoggedOff(LoggedOffCallback callback) { @@ -198,4 +319,96 @@ private void onLoggedOff(LoggedOffCallback callback) { isRunning = false; } + + // Depot Downloader Callbacks. + + @Override + public void onItemAdded(@NotNull DownloadItem item, int index) { + System.out.println("Depot Downloader: Item Added: " + item.getAppId() + ", index: " + index); + System.out.println(" ---- "); + } + + @Override + public void onItemRemoved(@NotNull DownloadItem item, int index) { + System.out.println("Depot Downloader: Item Removed: " + item.getAppId() + ", index: " + index); + System.out.println(" ---- "); + } + + @Override + public void onQueueCleared(@NotNull List previousItems) { + System.out.println("Depot Downloader: Queue size of " + previousItems.size() + " cleared"); + System.out.println(" ---- "); + } + + @Override + public void onDownloadStarted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download started for item: " + item.getAppId()); + System.out.println(" ---- "); + } + + @Override + public void onDownloadCompleted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download completed for item: " + item.getAppId()); + System.out.println(" ---- "); + } + + @Override + public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { + System.out.println("Depot Downloader: Download failed for item: " + item.getAppId()); + System.err.println(error.getMessage()); + System.out.println(" ---- "); + } + + @Override + public void onOverallProgress(@NotNull OverallProgress progress) { + System.out.println("Depot Downloader: Overall Progress"); + System.out.println("currentItem: " + progress.getCurrentItem()); + System.out.println("totalItems: " + progress.getTotalItems()); + System.out.println("totalBytesDownloaded: " + progress.getTotalBytesDownloaded()); + System.out.println("totalBytesExpected: " + progress.getTotalBytesExpected()); + System.out.println("status: " + progress.getStatus()); + System.out.println("percentComplete: " + progress.getPercentComplete()); + System.out.println(" ---- "); + } + + @Override + public void onDepotProgress(int depotId, @NotNull DepotProgress progress) { + System.out.println("Depot Downloader: Depot Progress"); + System.out.println("depotId: " + depotId); + System.out.println("depotId: " + progress.getDepotId()); + System.out.println("filesCompleted: " + progress.getFilesCompleted()); + System.out.println("totalFiles: " + progress.getTotalFiles()); + System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); + System.out.println("totalBytes: " + progress.getTotalBytes()); + System.out.println("status: " + progress.getStatus()); + System.out.println("percentComplete: " + progress.getPercentComplete()); + System.out.println(" ---- "); + } + + @Override + public void onFileProgress(int depotId, @NotNull String fileName, @NotNull FileProgress progress) { + System.out.println("Depot Downloader: File Progress"); + System.out.println("depotId: " + depotId); + System.out.println("fileName: " + fileName); + System.out.println("fileName: " + progress.getFileName()); + System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); + System.out.println("totalBytes: " + progress.getTotalBytes()); + System.out.println("chunksCompleted: " + progress.getChunksCompleted()); + System.out.println("totalChunks: " + progress.getTotalChunks()); + System.out.println("status: " + progress.getStatus()); + System.out.println("percentComplete: " + progress.getPercentComplete()); + System.out.println(" ---- "); + } + + @Override + public void onStatusUpdate(@NotNull String message) { + System.out.println("Depot Downloader: Status Message: " + message); + System.out.println(" ---- "); + } + + @Override + public void onAndroidEmulation(boolean value) { + System.out.println("Depot Downloader: Android Emulation: " + value); + System.out.println(" ---- "); + } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 1dc5b8bc..fa1f8579 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,7 @@ rootProject.name = "javasteam" -include(":javasteam-samples") include(":javasteam-cs") +include(":javasteam-deadlock") +include(":javasteam-depotdownloader") +include(":javasteam-dota2") +include(":javasteam-samples") include(":javasteam-tf") -include("javasteam-deadlock") -include("javasteam-dota2") diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index 9f7591f4..f77917e4 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -6,22 +6,23 @@ import `in`.dragonbra.javasteam.types.ChunkData import `in`.dragonbra.javasteam.types.DepotManifest import `in`.dragonbra.javasteam.util.SteamKitWebRequestException import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger -import `in`.dragonbra.javasteam.util.stream.MemoryStream import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.future.future +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout import okhttp3.HttpUrl import okhttp3.OkHttpClient import okhttp3.Request -import java.io.ByteArrayOutputStream +import okhttp3.coroutines.executeAsync +import java.io.ByteArrayInputStream import java.io.Closeable import java.io.IOException -import java.util.concurrent.* -import java.util.zip.DataFormatException +import java.util.concurrent.CompletableFuture import java.util.zip.ZipInputStream /** @@ -32,64 +33,76 @@ import java.util.zip.ZipInputStream */ class Client(steamClient: SteamClient) : Closeable { - private val httpClient: OkHttpClient = steamClient.configuration.httpClient - - private val defaultScope = CoroutineScope(Dispatchers.IO) + private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) companion object { - private val logger: Logger = LogManager.getLogger() + private val logger: Logger = LogManager.getLogger(Client::class.java) /** * Default timeout to use when making requests */ - var requestTimeout = 10000L + var requestTimeout = 10_000L /** * Default timeout to use when reading the response body */ - var responseBodyTimeout = 60000L + var responseBodyTimeout = 60_000L - @JvmStatic - @JvmOverloads - fun buildCommand( + private fun buildCommand( server: Server, command: String, query: String? = null, proxyServer: Server? = null, ): HttpUrl { - // TODO look into this to mimic SK's method. Should be able to remove if/else and only have the if. - val httpUrl: HttpUrl + val scheme = if (server.protocol == Server.ConnectionProtocol.HTTP) "http" else "https" + var host = server.vHost ?: server.host ?: "" + var port = server.port + var path = command + if (proxyServer != null && proxyServer.useAsProxy && proxyServer.proxyRequestPathTemplate != null) { - httpUrl = HttpUrl.Builder() - .scheme(if (proxyServer.protocol == Server.ConnectionProtocol.HTTP) "http" else "https") - .host(proxyServer.vHost) - .port(proxyServer.port) - .addPathSegment(server.vHost) - .addPathSegments(command) - .run { - query?.let { this.query(it) } ?: this - }.build() - } else { - httpUrl = HttpUrl.Builder() - .scheme(if (server.protocol == Server.ConnectionProtocol.HTTP) "http" else "https") - .host(server.vHost) - .port(server.port) - .addPathSegments(command) - .run { - query?.let { this.query(it) } ?: this - }.build() + val pathTemplate = proxyServer.proxyRequestPathTemplate!! + .replace("%host%", host) + .replace("%path%", "/$command") + + host = proxyServer.vHost ?: proxyServer.host ?: "" + port = proxyServer.port + path = pathTemplate + } + + val urlBuilder = HttpUrl.Builder() + .scheme(scheme) + .host(host) + .port(port) + .addPathSegments(path.trimStart('/')) + + query?.let { queryString -> + if (queryString.isNotEmpty()) { + val params = queryString.split("&") + for (param in params) { + val keyValue = param.split("=", limit = 2) + if (keyValue.size == 2) { + urlBuilder.addQueryParameter(keyValue[0], keyValue[1]) + } else if (keyValue.size == 1 && keyValue[0].isNotEmpty()) { + urlBuilder.addQueryParameter(keyValue[0], "") + } + } + } } - return httpUrl + return urlBuilder.build() } } + private val httpClient: OkHttpClient = steamClient.configuration.httpClient + /** * Disposes of this object. */ override fun close() { + scope.cancel() httpClient.connectionPool.evictAll() + httpClient.dispatcher.executorService.shutdown() } /** @@ -106,7 +119,6 @@ class Client(steamClient: SteamClient) : Closeable { * @exception IllegalArgumentException [server] was null. * @exception IOException A network error occurred when performing the request. * @exception SteamKitWebRequestException A network error occurred when performing the request. - * @exception DataFormatException When the data received is not as expected */ suspend fun downloadManifest( depotId: Int, @@ -116,8 +128,9 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): DepotManifest { + ): DepotManifest = withContext(Dispatchers.IO) { val manifestVersion = 5 + val url = if (manifestRequestCode > 0U) { "depot/$depotId/manifest/$manifestId/$manifestVersion/$manifestRequestCode" } else { @@ -128,8 +141,12 @@ class Client(steamClient: SteamClient) : Closeable { .url(buildCommand(server, url, cdnAuthToken, proxyServer)) .build() - return withTimeout(requestTimeout) { - val response = httpClient.newCall(request).execute() + logger.debug("Request URL is: $request") + + try { + val response = withTimeout(requestTimeout) { + httpClient.newCall(request).executeAsync() + } if (!response.isSuccessful) { throw SteamKitWebRequestException( @@ -138,93 +155,38 @@ class Client(steamClient: SteamClient) : Closeable { ) } - val depotManifest = withTimeout(responseBodyTimeout) { - val contentLength = response.header("Content-Length")?.toIntOrNull() + return@withContext withTimeout(responseBodyTimeout) { + response.use { resp -> + val responseBody = resp.body?.bytes() + ?: throw SteamKitWebRequestException("Response body is null") - if (contentLength == null) { - logger.debug("Manifest response does not have Content-Length, falling back to unbuffered read.") - } + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") + } - response.body.byteStream().use { inputStream -> - ByteArrayOutputStream().use { bs -> - val bytesRead = inputStream.copyTo(bs, contentLength ?: DEFAULT_BUFFER_SIZE) + // Decompress the zipped manifest data + ZipInputStream(ByteArrayInputStream(responseBody)).use { zipInputStream -> + zipInputStream.nextEntry + ?: throw SteamKitWebRequestException("Expected the zip to contain at least one file") - if (bytesRead != contentLength?.toLong()) { - throw DataFormatException("Length mismatch after downloading depot manifest! (was $bytesRead, but should be $contentLength)") - } + val manifestData = zipInputStream.readBytes() - val contentBytes = bs.toByteArray() - - MemoryStream(contentBytes).use { ms -> - ZipInputStream(ms).use { zip -> - var entryCount = 0 - while (zip.nextEntry != null) { - entryCount++ - } - if (entryCount > 1) { - logger.debug("Expected the zip to contain only one file") - } - } - } + val depotManifest = DepotManifest.deserialize(ByteArrayInputStream(manifestData)) - // Decompress the zipped manifest data - MemoryStream(contentBytes).use { ms -> - ZipInputStream(ms).use { zip -> - zip.nextEntry - DepotManifest.deserialize(zip) - } + if (depotKey != null) { + depotManifest.decryptFilenames(depotKey) } + + depotManifest } } } - - depotKey?.let { key -> - // if we have the depot key, decrypt the manifest filenames - depotManifest.decryptFilenames(key) - } - - depotManifest + } catch (e: Exception) { + logger.error("Failed to download manifest ${request.url}: ${e.message}", e) + throw e } } - /** - * Java Compat: - * Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided. - * @param depotId The id of the depot being accessed. - * @param manifestId The unique identifier of the manifest to be downloaded. - * @param manifestRequestCode The manifest request code for the manifest that is being downloaded. - * @param server The content server to connect to. - * @param depotKey The depot decryption key for the depot that will be downloaded. - * This is used for decrypting filenames (if needed) in depot manifests. - * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. - * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. - * @return A [DepotManifest] instance that contains information about the files present within a depot. - * @exception IllegalArgumentException [server] was null. - * @exception IOException A network error occurred when performing the request. - * @exception SteamKitWebRequestException A network error occurred when performing the request. - * @exception DataFormatException When the data received is not as expected - */ - @JvmOverloads - fun downloadManifestFuture( - depotId: Int, - manifestId: Long, - manifestRequestCode: Long, - server: Server, - depotKey: ByteArray? = null, - proxyServer: Server? = null, - cdnAuthToken: String? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadManifest( - depotId = depotId, - manifestId = manifestId, - manifestRequestCode = manifestRequestCode.toULong(), - server = server, - depotKey = depotKey, - proxyServer = proxyServer, - cdnAuthToken = cdnAuthToken, - ) - } - /** * Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided. * This function will also validate the length of the downloaded chunk with the value of [ChunkData.compressedLength], @@ -252,7 +214,7 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): Int { + ): Int = withContext(Dispatchers.IO) { require(chunk.chunkID != null) { "Chunk must have a ChunkID." } if (depotKey == null) { @@ -268,82 +230,118 @@ class Client(steamClient: SteamClient) : Closeable { val chunkID = Strings.toHex(chunk.chunkID) val url = "depot/$depotId/chunk/$chunkID" - val request: Request = if (ClientLancache.useLanCacheServer) { - ClientLancache.buildLancacheRequest(server, url, cdnAuthToken) + val request = if (ClientLancache.useLanCacheServer) { + ClientLancache.buildLancacheRequest(server = server, command = url, query = cdnAuthToken) } else { - Request.Builder().url(buildCommand(server, url, cdnAuthToken, proxyServer)).build() + val url = buildCommand(server = server, command = url, query = cdnAuthToken, proxyServer = proxyServer) + Request.Builder() + .url(url) + .build() } - withTimeout(requestTimeout) { - httpClient.newCall(request).execute() - }.use { response -> - if (!response.isSuccessful) { - throw SteamKitWebRequestException( - "Response status code does not indicate success: ${response.code} (${response.message})", - response - ) + try { + val response = withTimeout(requestTimeout) { + httpClient.newCall(request).executeAsync() } - var contentLength = chunk.compressedLength + response.use { resp -> + if (!resp.isSuccessful) { + throw SteamKitWebRequestException( + "Response status code does not indicate success: ${resp.code} (${resp.message})", + resp + ) + } - response.header("Content-Length")?.toLongOrNull()?.let { responseContentLength -> - contentLength = responseContentLength.toInt() + val contentLength = resp.body.contentLength().toInt() - // assert that lengths match only if the chunk has a length assigned. - if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { - throw IllegalStateException("Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})") + if (contentLength == 0) { + chunk.compressedLength } - } ?: run { - if (contentLength > 0) { - logger.debug("Response does not have Content-Length, falling back to chunk.compressedLength.") - } else { + + // Validate content length + if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { throw SteamKitWebRequestException( - "Response does not have Content-Length and chunk.compressedLength is not set.", - response + "Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})" ) } - } - // If no depot key is provided, stream into the destination buffer without renting - if (depotKey == null) { - val bytesRead = withTimeout(responseBodyTimeout) { - response.body.byteStream().use { input -> - input.readNBytesCompat(destination, 0, contentLength) - } + val responseBody = resp.body.bytes() + + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") } - if (bytesRead != contentLength) { - throw IOException("Length mismatch after downloading depot chunk! (was $bytesRead, but should be $contentLength)") + if (responseBody.size != contentLength) { + throw SteamKitWebRequestException( + "Length mismatch after downloading depot chunk! (was ${responseBody.size}, but should be $contentLength)" + ) } - return contentLength - } + if (depotKey == null) { + System.arraycopy(responseBody, 0, destination, 0, contentLength) + return@withContext contentLength + } - // We have to stream into a temporary buffer because a decryption will need to be performed - val buffer = ByteArray(contentLength) + return@withContext DepotChunk.process(chunk, responseBody, destination, depotKey) + } + } catch (e: Exception) { + logger.error("Failed to download a depot chunk ${request.url}: ${e.message}", e) + throw e + } + } - try { - val bytesRead = withTimeout(responseBodyTimeout) { - response.body.byteStream().use { input -> - input.readNBytesCompat(buffer, 0, contentLength) - } - } + // region Java Compatibility - if (bytesRead != contentLength) { - throw IOException("Length mismatch after downloading encrypted depot chunk! (was $bytesRead, but should be $contentLength)") - } + /** + * Java-compatible version of downloadManifest that returns a CompletableFuture. + * Downloads the depot manifest specified by the given manifest ID, and optionally decrypts the manifest's filenames if the depot decryption key has been provided. + * @param depotId The id of the depot being accessed. + * @param manifestId The unique identifier of the manifest to be downloaded. + * @param manifestRequestCode The manifest request code for the manifest that is being downloaded. + * @param server The content server to connect to. + * @param depotKey The depot decryption key for the depot that will be downloaded. + * This is used for decrypting filenames (if needed) in depot manifests. + * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. + * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. + * @return A CompletableFuture that will complete with a [DepotManifest] instance that contains information about the files present within a depot. + * @exception IllegalArgumentException [server] was null. + * @exception IOException A network error occurred when performing the request. + * @exception SteamKitWebRequestException A network error occurred when performing the request. + */ + @JvmOverloads + fun downloadManifestFuture( + depotId: Int, + manifestId: Long, + manifestRequestCode: Long, + server: Server, + depotKey: ByteArray? = null, + proxyServer: Server? = null, + cdnAuthToken: String? = null, + ): CompletableFuture { + val future = CompletableFuture() - // process the chunk immediately - return DepotChunk.process(chunk, buffer, destination, depotKey) - } catch (ex: Exception) { - logger.error("Failed to download a depot chunk ${request.url}", ex) - throw ex + scope.launch { + try { + val result = downloadManifest( + depotId = depotId, + manifestId = manifestId, + manifestRequestCode = manifestRequestCode.toULong(), + server = server, + depotKey = depotKey, + proxyServer = proxyServer, + cdnAuthToken = cdnAuthToken + ) + future.complete(result) + } catch (e: Exception) { + future.completeExceptionally(e) } } + + return future } /** - * Java Compat: + * Java-compatible version of downloadDepotChunk that returns a CompletableFuture. * Downloads the specified depot chunk, and optionally processes the chunk and verifies the checksum if the depot decryption key has been provided. * This function will also validate the length of the downloaded chunk with the value of [ChunkData.compressedLength], * if it has been assigned a value. @@ -357,7 +355,7 @@ class Client(steamClient: SteamClient) : Closeable { * This is used to process the chunk data. * @param proxyServer Optional content server marked as UseAsProxy which transforms the request. * @param cdnAuthToken CDN auth token for CDN content server endpoints if necessary. Get one with [SteamContent.getCDNAuthToken]. - * @return The total number of bytes written to [destination]. + * @return A CompletableFuture that will complete with the total number of bytes written to [destination]. * @exception IllegalArgumentException Thrown if the chunk's [ChunkData.chunkID] was null or if the [destination] buffer is too small. * @exception IllegalStateException Thrown if the downloaded data does not match the expected length. * @exception SteamKitWebRequestException A network error occurred when performing the request. @@ -371,15 +369,28 @@ class Client(steamClient: SteamClient) : Closeable { depotKey: ByteArray? = null, proxyServer: Server? = null, cdnAuthToken: String? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadDepotChunk( - depotId = depotId, - chunk = chunk, - server = server, - destination = destination, - depotKey = depotKey, - proxyServer = proxyServer, - cdnAuthToken = cdnAuthToken, - ) + ): CompletableFuture { + val future = CompletableFuture() + + scope.launch { + try { + val bytesWritten = downloadDepotChunk( + depotId = depotId, + chunk = chunk, + server = server, + destination = destination, + depotKey = depotKey, + proxyServer = proxyServer, + cdnAuthToken = cdnAuthToken + ) + future.complete(bytesWritten) + } catch (e: Exception) { + future.completeExceptionally(e) + } + } + + return future } + + // endregion } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt index 0108af47..0eb9e699 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt @@ -1,82 +1,126 @@ package `in`.dragonbra.javasteam.steam.cdn -import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.future.future -import okhttp3.HttpUrl +import kotlinx.coroutines.withContext +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Request import java.net.Inet4Address import java.net.Inet6Address import java.net.InetAddress -import java.util.concurrent.* /** * @author Lossy * @since 31/12/2024 */ -@Suppress("unused") object ClientLancache { - - private const val TRIGGER_DOMAIN: String = "lancache.steamcontent.com" - /** - * When set to true, will attempt to download from a Lancache instance on the LAN rather than going out to Steam's CDNs. + * When set to true, will attempt to download from a Lancache instance on the LAN + * rather than going out to Steam's CDNs. */ var useLanCacheServer: Boolean = false + private set + + private const val TRIGGER_DOMAIN = "lancache.steamcontent.com" /** - * Attempts to automatically resolve a Lancache on the local network. - * If detected, SteamKit will route all downloads through the cache rather than through Steam's CDN. + * Attempts to automatically resolve a Lancache on the local network. If detected, + * SteamKit will route all downloads through the cache rather than through Steam's CDN. * Will try to detect the Lancache through the poisoned DNS entry for lancache.steamcontent.com * - * This is a modified version from the original source : + * This is a modified version from the original source: * https://github.com/tpill90/lancache-prefill-common/blob/main/dotnet/LancacheIpResolver.cs */ - @JvmStatic - @JvmOverloads - fun detectLancacheServer(dispatcher: CoroutineDispatcher = Dispatchers.IO): CompletableFuture = - CoroutineScope(dispatcher).future { - val ipAddresses = InetAddress.getAllByName(TRIGGER_DOMAIN) - .filter { it is Inet4Address || it is Inet6Address } + suspend fun detectLancacheServer() { + withContext(Dispatchers.IO) { + try { + val addresses = InetAddress.getAllByName(TRIGGER_DOMAIN) + val ipAddresses = addresses.filter { address -> + address is Inet4Address || address is Inet6Address + } - useLanCacheServer = ipAddresses.any { isPrivateAddress(it) } + useLanCacheServer = ipAddresses.any { isPrivateAddress(it) } + } catch (_: Exception) { + useLanCacheServer = false + } } + } /** * Determines if an IP address is a private address, as specified in RFC1918 + * * @param toTest The IP address that will be tested - * @return true if the IP is a private address, false if it isn't private + * @return Returns true if the IP is a private address, false if it isn't private */ @JvmStatic fun isPrivateAddress(toTest: InetAddress): Boolean { - if (toTest.isLoopbackAddress) return true + if (toTest.isLoopbackAddress) { + return true + } + val bytes = toTest.address - return when (toTest) { - is Inet4Address -> when (bytes[0].toInt() and 0xFF) { + + // IPv4 + if (toTest is Inet4Address) { + // Convert signed byte to unsigned for comparison + val firstOctet = bytes[0].toInt() and 0xFF + + return when (firstOctet) { 10 -> true - 172 -> (bytes[1].toInt() and 0xFF) in 16..31 - 192 -> bytes[1].toInt() and 0xFF == 168 + 172 -> { + val secondOctet = bytes[1].toInt() and 0xFF + secondOctet in 16..<32 + } + + 192 -> { + val secondOctet = bytes[1].toInt() and 0xFF + secondOctet == 168 + } + else -> false } + } - is Inet6Address -> (bytes[0].toInt() and 0xFE) == 0xFC || toTest.isLinkLocalAddress - - else -> false + // IPv6 + if (toTest is Inet6Address) { + // Check for Unique Local Address (fc00::/7) and link-local + val firstByte = bytes[0].toInt() and 0xFF + return (firstByte and 0xFE) == 0xFC || toTest.isLinkLocalAddress } + + return false } - fun buildLancacheRequest(server: Server, command: String, query: String?): Request = Request.Builder() - .url( - HttpUrl.Builder() - .scheme("http") - .host("lancache.steamcontent.com") - .port(80) - .addPathSegments(command) - .query(query) - .build() - ) - .header("Host", server.host) - .header("User-Agent", "Valve/Steam HTTP Client 1.0") - .build() + /** + * Builds an HTTP request for Lancache with proper headers + * + * @param server The server to route the request through + * @param command The API command/path + * @param query Optional query parameters + * @return OkHttp Request object configured for Lancache + */ + fun buildLancacheRequest(server: Server, command: String, query: String? = null): Request { + val urlBuilder = "http://lancache.steamcontent.com:80".toHttpUrl().newBuilder() + .addPathSegments(command.trimStart('/')) + + query?.let { queryString -> + if (queryString.isNotEmpty()) { + val params = queryString.split("&") + for (param in params) { + val keyValue = param.split("=", limit = 2) + if (keyValue.size == 2) { + urlBuilder.addQueryParameter(keyValue[0], keyValue[1]) + } else if (keyValue.size == 1 && keyValue[0].isNotEmpty()) { + urlBuilder.addQueryParameter(keyValue[0], "") + } + } + } + } + + return Request.Builder() + .url(urlBuilder.build()) + .header("Host", server.host ?: "") + // User agent must match the Steam client in order for Lancache to correctly identify and cache Valve's CDN content + .header("User-Agent", "Valve/Steam HTTP Client 1.0") + .build() + } } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt deleted file mode 100644 index ee377dd2..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientPool.kt +++ /dev/null @@ -1,142 +0,0 @@ -package `in`.dragonbra.javasteam.steam.cdn - -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Job -import kotlinx.coroutines.async -import kotlinx.coroutines.cancel -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import kotlinx.coroutines.launch -import java.util.concurrent.ConcurrentLinkedDeque -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit - -/** - * [ClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed - */ -class ClientPool(internal val steamClient: SteamClient, private val appId: Int, private val parentScope: CoroutineScope) { - - companion object { - private const val SERVER_ENDPOINT_MIN_SIZE = 8 - } - - val cdnClient: Client = Client(steamClient) - - var proxyServer: Server? = null - private set - - private val activeConnectionPool = ConcurrentLinkedDeque() - - private val availableServerEndpoints = ConcurrentLinkedQueue() - - private val populatePoolEvent = CountDownLatch(1) - - private val monitorJob: Job - - private val logger: Logger = LogManager.getLogger() - - init { - monitorJob = parentScope.launch { connectionPoolMonitor().await() } - } - - fun shutdown() { - monitorJob.cancel() - } - - private fun fetchBootstrapServerList(): Deferred?> = parentScope.async { - return@async try { - steamClient.getHandler(SteamContent::class.java)?.getServersForSteamPipe(parentScope = parentScope)?.await() - } catch (ex: Exception) { - logger.error("Failed to retrieve content server list", ex) - - null - } - } - - private fun connectionPoolMonitor() = parentScope.async { - var didPopulate = false - - while (isActive) { - populatePoolEvent.await(1, TimeUnit.SECONDS) - - if (availableServerEndpoints.size < SERVER_ENDPOINT_MIN_SIZE && steamClient.isConnected) { - val servers = fetchBootstrapServerList().await() - - if (servers.isNullOrEmpty()) { - logger.error("Servers is empty or null, exiting connection pool monitor") - parentScope.cancel() - return@async - } - - proxyServer = servers.find { it.useAsProxy } - - val weightedCdnServers = servers - .filter { server -> - val isEligibleForApp = server.allowedAppIds.isEmpty() || appId in server.allowedAppIds - isEligibleForApp && (server.type == "SteamCache" || server.type == "CDN") - } - .sortedBy { it.weightedLoad } - - for (server in weightedCdnServers) { - repeat(server.numEntries) { - availableServerEndpoints.offer(server) - } - } - - didPopulate = true - } else if (availableServerEndpoints.isEmpty() && !steamClient.isConnected && didPopulate) { - logger.error("Available server endpoints is empty and steam is not connected, exiting connection pool monitor") - - parentScope.cancel() - - return@async - } - } - } - - private fun buildConnection(): Deferred = parentScope.async { - return@async try { - if (availableServerEndpoints.size < SERVER_ENDPOINT_MIN_SIZE) { - populatePoolEvent.countDown() - } - - var output: Server? = null - - while (isActive && availableServerEndpoints.poll().also { output = it } == null) { - delay(1000) - } - - output - } catch (e: Exception) { - logger.error("Failed to build connection", e) - - null - } - } - - internal fun getConnection(): Deferred = parentScope.async { - return@async try { - val server = activeConnectionPool.poll() ?: buildConnection().await() - server - } catch (e: Exception) { - logger.error("Failed to get/build connection", e) - - null - } - } - - internal fun returnConnection(server: Server?) { - server?.let { activeConnectionPool.push(it) } - } - - @Suppress("unused") - internal fun returnBrokenConnection(server: Server?) { - // Broken connections are not returned to the pool - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt index 6285449f..db744e3c 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/DepotChunk.kt @@ -51,15 +51,16 @@ object DepotChunk { require(iv.size == ivBytesRead) { "Failed to decrypt depot chunk iv (${iv.size} != $ivBytesRead)" } + val aes = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) + aes.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) + // With CBC and padding, the decrypted size will always be smaller val buffer = ByteArray(data.size - iv.size) - val cbcCipher = Cipher.getInstance("AES/CBC/PKCS7Padding", CryptoHelper.SEC_PROV) - cbcCipher.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) val writtenDecompressed: Int try { - val written = cbcCipher.doFinal(data, iv.size, data.size - iv.size, buffer) + val written = aes.doFinal(data, iv.size, data.size - iv.size, buffer) // Per SK: // Steam client checks for like 20 bytes for pkzip, and 22 bytes for vzip, diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt index 7b9aa104..82a2035f 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Server.kt @@ -1,24 +1,35 @@ package `in`.dragonbra.javasteam.steam.cdn +import java.net.InetSocketAddress + /** * Represents a single Steam3 'Steampipe' content server. */ -class Server @JvmOverloads constructor( - protocol: ConnectionProtocol = ConnectionProtocol.HTTP, - host: String, - vHost: String, - port: Int, - type: String? = null, - sourceID: Int = 0, - cellID: Int = 0, - load: Int = 0, - weightedLoad: Float = 0f, - numEntries: Int = 0, - steamChinaOnly: Boolean = false, - useAsProxy: Boolean = false, - proxyRequestPathTemplate: String? = null, - allowedAppIds: IntArray = IntArray(0), -) { +class Server { + + companion object { + /** + * Creates a Server from an InetSocketAddress. + */ + @JvmStatic + fun fromInetSocketAddress(endPoint: InetSocketAddress): Server = Server().apply { + protocol = if (endPoint.port == 443) ConnectionProtocol.HTTPS else ConnectionProtocol.HTTP + host = endPoint.address.hostAddress + vHost = endPoint.address.hostAddress + port = endPoint.port + } + + /** + * Creates a Server from hostname and port. + */ + @JvmStatic + fun fromHostAndPort(hostname: String, port: Int): Server = Server().apply { + protocol = if (port == 443) ConnectionProtocol.HTTPS else ConnectionProtocol.HTTP + host = hostname + vHost = hostname + this.port = port + } + } /** * The protocol used to connect to this server @@ -38,86 +49,85 @@ class Server @JvmOverloads constructor( /** * Gets the supported connection protocol of the server. */ - var protocol = protocol + var protocol: ConnectionProtocol = ConnectionProtocol.HTTP internal set /** * Gets the hostname of the server. */ - var host = host + var host: String? = null internal set /** * Gets the virtual hostname of the server. */ - var vHost = vHost + var vHost: String? = null internal set /** * Gets the port of the server. */ - var port = port + var port: Int = 0 internal set /** * Gets the type of the server. */ - var type = type + var type: String? = null internal set /** * Gets the SourceID this server belongs to. */ - @Suppress("unused") - var sourceID = sourceID + var sourceId: Int = 0 internal set /** * Gets the CellID this server belongs to. */ - var cellID = cellID + var cellId: Int = 0 internal set /** * Gets the load value associated with this server. */ - var load = load + var load: Int = 0 internal set /** * Gets the weighted load. */ - var weightedLoad = weightedLoad + var weightedLoad: Float = 0F internal set /** * Gets the number of entries this server is worth. */ - var numEntries = numEntries + var numEntries: Int = 0 internal set /** * Gets the flag whether this server is for Steam China only. */ - var steamChinaOnly = steamChinaOnly + var steamChinaOnly: Boolean = false internal set /** * Gets the download proxy status. */ - var useAsProxy = useAsProxy + var useAsProxy: Boolean = false internal set /** * Gets the transformation template applied to request paths. */ - var proxyRequestPathTemplate = proxyRequestPathTemplate + var proxyRequestPathTemplate: String? = null internal set /** * Gets the list of app ids this server can be used with. */ - var allowedAppIds = allowedAppIds + var allowedAppIds: IntArray = intArrayOf() internal set /** diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt deleted file mode 100644 index 2004de2b..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ChunkMatch.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.ChunkData - -data class ChunkMatch( - val oldChunk: ChunkData, - val newChunk: ChunkData, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt deleted file mode 100644 index 2a638d65..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ContentDownloader.kt +++ /dev/null @@ -1,724 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.enums.EDepotFileFlag -import `in`.dragonbra.javasteam.enums.EResult -import `in`.dragonbra.javasteam.steam.cdn.ClientPool -import `in`.dragonbra.javasteam.steam.cdn.Server -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSProductInfo -import `in`.dragonbra.javasteam.steam.handlers.steamapps.PICSRequest -import `in`.dragonbra.javasteam.steam.handlers.steamapps.SteamApps -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient -import `in`.dragonbra.javasteam.types.ChunkData -import `in`.dragonbra.javasteam.types.DepotManifest -import `in`.dragonbra.javasteam.types.FileData -import `in`.dragonbra.javasteam.types.KeyValue -import `in`.dragonbra.javasteam.util.Adler32 -import `in`.dragonbra.javasteam.util.SteamKitWebRequestException -import `in`.dragonbra.javasteam.util.Strings -import `in`.dragonbra.javasteam.util.Utils -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Deferred -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.future.future -import kotlinx.coroutines.isActive -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit -import java.io.File -import java.io.FileInputStream -import java.io.FileOutputStream -import java.io.RandomAccessFile -import java.nio.ByteBuffer -import java.nio.file.Paths -import java.time.Instant -import java.time.temporal.ChronoUnit -import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentLinkedQueue - -@Suppress("unused", "SpellCheckingInspection") -class ContentDownloader(val steamClient: SteamClient) { - - companion object { - private const val HTTP_UNAUTHORIZED = 401 - private const val HTTP_FORBIDDEN = 403 - private const val HTTP_NOT_FOUND = 404 - private const val SERVICE_UNAVAILABLE = 503 - - internal const val INVALID_APP_ID = Int.MAX_VALUE - internal const val INVALID_MANIFEST_ID = Long.MAX_VALUE - - private val logger: Logger = LogManager.getLogger(ContentDownloader::class.java) - } - - private val defaultScope = CoroutineScope(Dispatchers.IO) - - private fun requestDepotKey( - appId: Int, - depotId: Int, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.getDepotDecryptionKey(depotId, appId)?.await() - - return@async Pair(callback?.result ?: EResult.Fail, callback?.depotKey) - } - - private fun getDepotManifestId( - app: PICSProductInfo, - depotId: Int, - branchId: String, - parentScope: CoroutineScope, - ): Deferred> = parentScope.async { - val depot = app.keyValues["depots"][depotId.toString()] - if (depot == KeyValue.INVALID) { - logger.error("Could not find depot $depotId of ${app.id}") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val manifest = depot["manifests"][branchId] - if (manifest != KeyValue.INVALID) { - return@async Pair(app.id, manifest["gid"].asLong()) - } - - val depotFromApp = depot["depotfromapp"].asInteger(INVALID_APP_ID) - if (depotFromApp == app.id || depotFromApp == INVALID_APP_ID) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - val innerApp = getAppInfo(depotFromApp, parentScope).await() - if (innerApp == null) { - logger.error("Failed to find manifest of app ${app.id} within depot $depotId on branch $branchId") - return@async Pair(app.id, INVALID_MANIFEST_ID) - } - - return@async getDepotManifestId(innerApp, depotId, branchId, parentScope).await() - } - - private fun getAppDirName(app: PICSProductInfo): String { - val installDirKeyValue = app.keyValues["config"]["installdir"] - - return if (installDirKeyValue != KeyValue.INVALID) installDirKeyValue.value!! else app.id.toString() - } - - private fun getAppInfo( - appId: Int, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - val steamApps = steamClient.getHandler(SteamApps::class.java) - val callback = steamApps?.picsGetProductInfo(PICSRequest(appId))?.await() - val apps = callback?.results?.flatMap { it.apps.values } - - if (apps.isNullOrEmpty()) { - logger.error("Received empty apps list in PICSProductInfo response for $appId") - return@async null - } - - if (apps.size > 1) { - logger.debug("Received ${apps.size} apps from PICSProductInfo for $appId, using first result") - } - - return@async apps.first() - } - - /** - * Kotlin coroutines version - */ - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope = defaultScope, - ): Deferred = parentScope.async { - downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = onDownloadProgress, - scope = parentScope - ) - } - - /** - * Java-friendly version that returns a CompletableFuture - */ - @JvmOverloads - fun downloadApp( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - progressCallback: ProgressCallback? = null, - ): CompletableFuture = defaultScope.future { - return@future downloadAppInternal( - appId = appId, - depotId = depotId, - installPath = installPath, - stagingPath = stagingPath, - branch = branch, - maxDownloads = maxDownloads, - onDownloadProgress = progressCallback?.let { callback -> { progress -> callback.onProgress(progress) } }, - scope = defaultScope - ) - } - - private suspend fun downloadAppInternal( - appId: Int, - depotId: Int, - installPath: String, - stagingPath: String, - branch: String = "public", - maxDownloads: Int = 8, - onDownloadProgress: ((Float) -> Unit)? = null, - scope: CoroutineScope, - ): Boolean { - if (!scope.isActive) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - return false - } - - val cdnPool = ClientPool(steamClient, appId, scope) - - val shiftedAppId: Int - val manifestId: Long - val appInfo = getAppInfo(appId, scope).await() - - if (appInfo == null) { - logger.error("Could not retrieve PICSProductInfo of $appId") - return false - } - - getDepotManifestId(appInfo, depotId, branch, scope).await().apply { - shiftedAppId = first - manifestId = second - } - - val depotKeyResult = requestDepotKey(shiftedAppId, depotId, scope).await() - - if (depotKeyResult.first != EResult.OK || depotKeyResult.second == null) { - logger.error("Depot key request for $appId failed with result ${depotKeyResult.first}") - return false - } - - val depotKey = depotKeyResult.second!! - - var newProtoManifest = steamClient.configuration.depotManifestProvider.fetchManifest(depotId, manifestId) - var oldProtoManifest = steamClient.configuration.depotManifestProvider.fetchLatestManifest(depotId) - - if (oldProtoManifest?.manifestGID == manifestId) { - oldProtoManifest = null - } - - // In case we have an early exit, this will force equiv of verifyall next run. - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, INVALID_MANIFEST_ID) - - try { - if (newProtoManifest == null) { - newProtoManifest = - downloadFilesManifestOf(shiftedAppId, depotId, manifestId, branch, depotKey, cdnPool, scope).await() - } else { - logger.debug("Already have manifest $manifestId for depot $depotId.") - } - - if (newProtoManifest == null) { - logger.error("Failed to retrieve files manifest for app: $shiftedAppId depot: $depotId manifest: $manifestId branch: $branch") - return false - } - - if (!scope.isActive) { - return false - } - - val downloadCounter = GlobalDownloadCounter() - val installDir = Paths.get(installPath, getAppDirName(appInfo)).toString() - val stagingDir = Paths.get(stagingPath, getAppDirName(appInfo)).toString() - val depotFileData = DepotFilesData( - depotDownloadInfo = DepotDownloadInfo(depotId, shiftedAppId, manifestId, branch, installDir, depotKey), - depotCounter = DepotDownloadCounter( - completeDownloadSize = newProtoManifest.totalUncompressedSize - ), - stagingDir = stagingDir, - manifest = newProtoManifest, - previousManifest = oldProtoManifest - ) - - downloadDepotFiles(cdnPool, downloadCounter, depotFileData, maxDownloads, onDownloadProgress, scope).await() - - steamClient.configuration.depotManifestProvider.setLatestManifestId(depotId, manifestId) - - cdnPool.shutdown() - - // delete the staging directory of this app - File(stagingDir).deleteRecursively() - - logger.debug( - "Depot $depotId - Downloaded ${depotFileData.depotCounter.depotBytesCompressed} " + - "bytes (${depotFileData.depotCounter.depotBytesUncompressed} bytes uncompressed)" - ) - - return true - } catch (e: CancellationException) { - logger.error("App $appId was not completely downloaded. Operation was canceled.") - - return false - } catch (e: Exception) { - logger.error("Error occurred while downloading app $shiftedAppId", e) - - return false - } - } - - private fun downloadDepotFiles( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - maxDownloads: Int, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!parentScope.isActive) { - return@async - } - - depotFilesData.manifest.files.forEach { file -> - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - if (file.flags.contains(EDepotFileFlag.Directory)) { - File(fileFinalPath).mkdirs() - File(fileStagingPath).mkdirs() - } else { - // Some manifests don't explicitly include all necessary directories - File(fileFinalPath).parentFile.mkdirs() - File(fileStagingPath).parentFile.mkdirs() - } - } - - logger.debug("Downloading depot ${depotFilesData.depotDownloadInfo.depotId}") - - val files = depotFilesData.manifest.files.filter { !it.flags.contains(EDepotFileFlag.Directory) }.toTypedArray() - val networkChunkQueue = ConcurrentLinkedQueue>() - - val downloadSemaphore = Semaphore(maxDownloads) - files.map { file -> - async { - downloadSemaphore.withPermit { - downloadDepotFile(depotFilesData, file, networkChunkQueue, onDownloadProgress, parentScope).await() - } - } - }.awaitAll() - - networkChunkQueue.map { (fileStreamData, fileData, chunk) -> - async { - downloadSemaphore.withPermit { - downloadSteam3DepotFileChunk( - cdnPool = cdnPool, - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = fileData, - fileStreamData = fileStreamData, - chunk = chunk, - onDownloadProgress = onDownloadProgress, - parentScope = parentScope - ).await() - } - } - }.awaitAll() - - // Check for deleted files if updating the depot. - depotFilesData.previousManifest?.apply { - val previousFilteredFiles = files.asSequence().map { it.fileName }.toMutableSet() - - // Of the list of files in the previous manifest, remove any file names that exist in the current set of all file names - previousFilteredFiles.removeAll(depotFilesData.manifest.files.map { it.fileName }.toSet()) - - for (existingFileName in previousFilteredFiles) { - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, existingFileName).toString() - - if (!File(fileFinalPath).exists()) { - continue - } - - File(fileFinalPath).delete() - logger.debug("Deleted $fileFinalPath") - } - } - } - - private fun downloadDepotFile( - depotFilesData: DepotFilesData, - file: FileData, - networkChunkQueue: ConcurrentLinkedQueue>, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depotDownloadCounter = depotFilesData.depotCounter - val oldManifestFile = depotFilesData.previousManifest?.files?.find { it.fileName == file.fileName } - - val fileFinalPath = Paths.get(depotFilesData.depotDownloadInfo.installDir, file.fileName).toString() - val fileStagingPath = Paths.get(depotFilesData.stagingDir, file.fileName).toString() - - // This may still exist if the previous run exited before cleanup - File(fileStagingPath).takeIf { it.exists() }?.delete() - - val neededChunks: MutableList - val fi = File(fileFinalPath) - val fileDidExist = fi.exists() - - if (!fileDidExist) { - // create new file. need all chunks - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - } - - neededChunks = file.chunks.toMutableList() - } else { - // open existing - if (oldManifestFile != null) { - neededChunks = mutableListOf() - - val hashMatches = oldManifestFile.fileHash.contentEquals(file.fileHash) - if (!hashMatches) { - logger.debug("Validating $fileFinalPath") - - val matchingChunks = mutableListOf() - - for (chunk in file.chunks) { - val oldChunk = oldManifestFile.chunks.find { it.chunkID.contentEquals(chunk.chunkID) } - if (oldChunk != null) { - matchingChunks.add(ChunkMatch(oldChunk, chunk)) - } else { - neededChunks.add(chunk) - } - } - - val orderedChunks = matchingChunks.sortedBy { it.oldChunk.offset } - - val copyChunks = mutableListOf() - - FileInputStream(fileFinalPath).use { fsOld -> - for (match in orderedChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - val adler = Adler32.calculate(tmp) - if (adler != match.oldChunk.checksum) { - neededChunks.add(match.newChunk) - } else { - copyChunks.add(match) - } - } - } - - if (neededChunks.isNotEmpty()) { - File(fileFinalPath).renameTo(File(fileStagingPath)) - - FileInputStream(fileStagingPath).use { fsOld -> - FileOutputStream(fileFinalPath).use { fs -> - fs.channel.truncate(file.totalSize) - - for (match in copyChunks) { - fsOld.channel.position(match.oldChunk.offset) - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - fsOld.readNBytesCompat(tmp, 0, tmp.size) - - fs.channel.position(match.newChunk.offset) - fs.write(tmp) - } - } - } - - File(fileStagingPath).delete() - } - } - } else { - // No old manifest or file not in old manifest. We must validate. - RandomAccessFile(fileFinalPath, "rw").use { fs -> - if (fi.length() != file.totalSize) { - fs.channel.truncate(file.totalSize) - } - - logger.debug("Validating $fileFinalPath") - neededChunks = Utils.validateSteam3FileChecksums( - fs, - file.chunks.sortedBy { it.offset }.toTypedArray() - ) - } - } - - if (neededChunks.isEmpty()) { - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += file.totalSize - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - - return@async - } - - val sizeOnDisk = file.totalSize - neededChunks.sumOf { it.uncompressedLength.toLong() } - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += sizeOnDisk - } - - onDownloadProgress?.apply { - val totalPercent = - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - this(totalPercent) - } - } - - val fileIsExecutable = file.flags.contains(EDepotFileFlag.Executable) - if (fileIsExecutable && - (!fileDidExist || oldManifestFile == null || !oldManifestFile.flags.contains(EDepotFileFlag.Executable)) - ) { - File(fileFinalPath).setExecutable(true) - } else if (!fileIsExecutable && oldManifestFile != null && oldManifestFile.flags.contains(EDepotFileFlag.Executable)) { - File(fileFinalPath).setExecutable(false) - } - - val fileStreamData = FileStreamData( - fileStream = null, - fileLock = Semaphore(1), - chunksToDownload = neededChunks.size - ) - - for (chunk in neededChunks) { - networkChunkQueue.add(Triple(fileStreamData, file, chunk)) - } - } - - private fun downloadSteam3DepotFileChunk( - cdnPool: ClientPool, - downloadCounter: GlobalDownloadCounter, - depotFilesData: DepotFilesData, - file: FileData, - fileStreamData: FileStreamData, - chunk: ChunkData, - onDownloadProgress: ((Float) -> Unit)? = null, - parentScope: CoroutineScope, - ) = parentScope.async { - if (!isActive) { - return@async - } - - val depot = depotFilesData.depotDownloadInfo - val depotDownloadCounter = depotFilesData.depotCounter - - val chunkID = Strings.toHex(chunk.chunkID) - - var outputChunkData = ByteArray(chunk.uncompressedLength) - var writtenBytes = 0 - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - outputChunkData = ByteArray(chunk.uncompressedLength) - writtenBytes = cdnPool.cdnClient.downloadDepotChunk( - depotId = depot.depotId, - chunk = chunk, - server = connection!!, - destination = outputChunkData, - depotKey = depot.depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - when (e.statusCode) { - HTTP_UNAUTHORIZED, HTTP_FORBIDDEN -> { - logger.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") - break - } - - else -> logger.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") - } - } catch (e: NoClassDefFoundError) { - // Zstd is a 'compileOnly' dependency. - throw CancellationException(e.message) - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered unexpected error downloading chunk $chunkID", e) - } - } while (isActive && writtenBytes <= 0) - - if (writtenBytes <= 0) { - logger.error("Failed to find any server with chunk $chunkID for depot ${depot.depotId}. Aborting.") - throw CancellationException("Failed to download chunk") - } - - try { - fileStreamData.fileLock.acquire() - - if (fileStreamData.fileStream == null) { - val fileFinalPath = Paths.get(depot.installDir, file.fileName).toString() - val randomAccessFile = RandomAccessFile(fileFinalPath, "rw") - fileStreamData.fileStream = randomAccessFile.channel - } - - fileStreamData.fileStream?.position(chunk.offset) - fileStreamData.fileStream?.write(ByteBuffer.wrap(outputChunkData, 0, writtenBytes)) - } finally { - fileStreamData.fileLock.release() - } - - val remainingChunks = synchronized(fileStreamData) { - --fileStreamData.chunksToDownload - } - if (remainingChunks <= 0) { - fileStreamData.fileStream?.close() - } - - var sizeDownloaded: Long - synchronized(depotDownloadCounter) { - sizeDownloaded = depotDownloadCounter.sizeDownloaded + outputChunkData.size - depotDownloadCounter.sizeDownloaded = sizeDownloaded - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - - onDownloadProgress?.invoke( - depotFilesData.depotCounter.sizeDownloaded.toFloat() / depotFilesData.depotCounter.completeDownloadSize - ) - } - - private fun downloadFilesManifestOf( - appId: Int, - depotId: Int, - manifestId: Long, - branch: String, - depotKey: ByteArray, - cdnPool: ClientPool, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - if (!isActive) { - return@async null - } - - var depotManifest: DepotManifest? = null - var manifestRequestCode = 0UL - var manifestRequestCodeExpiration = Instant.MIN - - do { - var connection: Server? = null - - try { - connection = cdnPool.getConnection().await() - - if (connection == null) continue - - val now = Instant.now() - - // In order to download this manifest, we need the current manifest request code - // The manifest request code is only valid for a specific period of time - if (manifestRequestCode == 0UL || now >= manifestRequestCodeExpiration) { - val steamContent = steamClient.getHandler(SteamContent::class.java)!! - - manifestRequestCode = steamContent.getManifestRequestCode( - depotId = depotId, - appId = appId, - manifestId = manifestId, - branch = branch, - parentScope = parentScope - ).await() - - // This code will hopefully be valid for one period following the issuing period - manifestRequestCodeExpiration = now.plus(5, ChronoUnit.MINUTES) - - // If we could not get the manifest code, this is a fatal error - if (manifestRequestCode == 0UL) { - throw CancellationException("No manifest request code was returned for manifest $manifestId in depot $depotId") - } - } - - depotManifest = cdnPool.cdnClient.downloadManifest( - depotId = depotId, - manifestId = manifestId, - manifestRequestCode = manifestRequestCode, - server = connection, - depotKey = depotKey, - proxyServer = cdnPool.proxyServer - ) - - cdnPool.returnConnection(connection) - } catch (e: CancellationException) { - logger.error("Connection timeout downloading depot manifest $depotId $manifestId") - - return@async null - } catch (e: SteamKitWebRequestException) { - cdnPool.returnBrokenConnection(connection) - - val statusName = when (e.statusCode) { - HTTP_UNAUTHORIZED -> HTTP_UNAUTHORIZED::class.java.name - HTTP_FORBIDDEN -> HTTP_FORBIDDEN::class.java.name - HTTP_NOT_FOUND -> HTTP_NOT_FOUND::class.java.name - SERVICE_UNAVAILABLE -> SERVICE_UNAVAILABLE::class.java.name - else -> null - } - - logger.error( - "Downloading of manifest $manifestId failed for depot $depotId with " + - if (statusName != null) { - "response of $statusName(${e.statusCode})" - } else { - "status code of ${e.statusCode}" - } - ) - - return@async null - } catch (e: Exception) { - cdnPool.returnBrokenConnection(connection) - - logger.error("Encountered error downloading manifest for depot $depotId $manifestId", e) - - return@async null - } - } while (isActive && depotManifest == null) - - if (depotManifest == null) { - throw CancellationException("Unable to download manifest $manifestId for depot $depotId") - } - - val newProtoManifest = depotManifest - steamClient.configuration.depotManifestProvider.updateManifest(newProtoManifest) - - return@async newProtoManifest - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt deleted file mode 100644 index dd31905a..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadCounter.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -data class DepotDownloadCounter( - var completeDownloadSize: Long = 0, - var sizeDownloaded: Long = 0, - var depotBytesCompressed: Long = 0, - var depotBytesUncompressed: Long = 0, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt deleted file mode 100644 index ff31b973..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotDownloadInfo.kt +++ /dev/null @@ -1,11 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -@Suppress("ArrayInDataClass") -data class DepotDownloadInfo( - val depotId: Int, - val appId: Int, - val manifestId: Long, - val branch: String, - val installDir: String, - val depotKey: ByteArray?, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt deleted file mode 100644 index f2e01726..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/DepotFilesData.kt +++ /dev/null @@ -1,11 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -data class DepotFilesData( - val depotDownloadInfo: DepotDownloadInfo, - val depotCounter: DepotDownloadCounter, - val stagingDir: String, - val manifest: DepotManifest, - val previousManifest: DepotManifest?, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt deleted file mode 100644 index ec7201b6..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileManifestProvider.kt +++ /dev/null @@ -1,210 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest -import `in`.dragonbra.javasteam.util.compat.readNBytesCompat -import `in`.dragonbra.javasteam.util.log.LogManager -import `in`.dragonbra.javasteam.util.log.Logger -import `in`.dragonbra.javasteam.util.stream.MemoryStream -import java.io.ByteArrayOutputStream -import java.io.File -import java.io.IOException -import java.nio.ByteBuffer -import java.nio.file.Files -import java.nio.file.Path -import java.util.zip.CRC32 -import java.util.zip.ZipEntry -import java.util.zip.ZipInputStream -import java.util.zip.ZipOutputStream - -/** - * Depot manifest provider that stores depot manifests in a zip file. - * @constructor Instantiates a [FileManifestProvider] object. - * @param file the file that will store the depot manifests - * - * @author Oxters - * @since 2024-11-07 - */ -@Suppress("unused") -class FileManifestProvider(private val file: Path) : IManifestProvider { - - /** - * Instantiates a [FileManifestProvider] object. - * @param file the file that will store the depot manifests. - */ - constructor(file: File) : this(file.toPath()) - - /** - * Instantiates a [FileManifestProvider] object. - * @param filename the filename that will store the depot manifests. - */ - constructor(filename: String) : this(Path.of(filename)) - - init { - require(file.fileName.toString().isNotBlank()) { "FileName must not be blank" } - } - - companion object { - private val logger: Logger = LogManager.getLogger(FileManifestProvider::class.java) - - private fun getLatestEntryName(depotID: Int): String = "$depotID${File.separator}latest" - - private fun getEntryName(depotID: Int, manifestID: Long): String = "$depotID${File.separator}$manifestID.bin" - - private fun seekToEntry(zipStream: ZipInputStream, entryName: String): ZipEntry? { - var zipEntry: ZipEntry? - - do { - zipEntry = zipStream.nextEntry - if (zipEntry?.name.equals(entryName, true)) { - break - } - } while (zipEntry != null) - - return zipEntry - } - - private fun copyZip(from: ZipInputStream, to: ZipOutputStream, vararg excludeEntries: String) { - var entry = from.nextEntry - - while (entry != null) { - if (!excludeEntries.contains(entry.name) && (entry.isDirectory || (!entry.isDirectory && entry.size > 0))) { - to.putNextEntry(entry) - - if (!entry.isDirectory) { - val entryBytes = ByteArray(entry.size.toInt()) - - from.readNBytesCompat(entryBytes, 0, entryBytes.size) - to.write(entryBytes) - } - - to.closeEntry() - } - - entry = from.nextEntry - } - } - - private fun zipUncompressed(zip: ZipOutputStream, entryName: String, bytes: ByteArray) { - val entry = ZipEntry(entryName).apply { - method = ZipEntry.STORED - size = bytes.size.toLong() - compressedSize = bytes.size.toLong() - crc = CRC32().run { - update(bytes) - value - } - } - - zip.putNextEntry(entry) - zip.write(bytes) - zip.closeEntry() - } - } - - override fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? = runCatching { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zip -> - seekToEntry(zip, getEntryName(depotID, manifestID))?.let { - if (it.size > 0) { - DepotManifest.deserialize(zip) - } else { - null - } - } - } - } - }.fold( - onSuccess = { it }, - onFailure = { error -> - when (error) { - is NoSuchFileException -> logger.debug("File doesn't exist") - else -> logger.error("Unknown error occurred", error) - } - - null - } - ) - - override fun fetchLatestManifest(depotID: Int): DepotManifest? = runCatching { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zip -> - seekToEntry(zip, getLatestEntryName(depotID))?.let { idEntry -> - if (idEntry.size > 0) { - ByteBuffer.wrap(zip.readNBytesCompat(idEntry.size.toInt())).getLong() - } else { - null - } - } - } - }?.let { manifestId -> - fetchManifest(depotID, manifestId) - } - }.fold( - onSuccess = { it }, - onFailure = { error -> - when (error) { - is NoSuchFileException -> logger.debug("File doesn't exist") - else -> logger.error("Unknown error occurred", error) - } - - null - } - ) - - override fun setLatestManifestId(depotID: Int, manifestID: Long) { - ByteArrayOutputStream().use { bs -> - ZipOutputStream(bs).use { zip -> - // copy old file only if it exists - if (Files.exists(file)) { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zs -> - copyZip(zs, zip, getLatestEntryName(depotID)) - } - } - } - // write manifest id as uncompressed data - ByteBuffer.allocate(Long.SIZE_BYTES).apply { - putLong(manifestID) - zipUncompressed(zip, getLatestEntryName(depotID), array()) - } - } - // save all data to the file - try { - Files.newOutputStream(file).use { fos -> - fos.write(bs.toByteArray()) - } - } catch (e: IOException) { - logger.error("Failed to write manifest ID to file ${file.fileName}", e) - } - } - } - - override fun updateManifest(manifest: DepotManifest) { - ByteArrayOutputStream().use { bs -> - ZipOutputStream(bs).use { zip -> - // copy old file only if it exists - if (Files.exists(file)) { - Files.newInputStream(file).use { fis -> - ZipInputStream(fis).use { zs -> - copyZip(zs, zip, getEntryName(manifest.depotID, manifest.manifestGID)) - } - } - } - // add manifest as uncompressed data - MemoryStream().use { ms -> - manifest.serialize(ms.asOutputStream()) - - zipUncompressed(zip, getEntryName(manifest.depotID, manifest.manifestGID), ms.toByteArray()) - } - } - // save all data to the file - try { - Files.newOutputStream(file).use { fos -> - fos.write(bs.toByteArray()) - } - } catch (e: IOException) { - logger.error("Failed to write manifest to file ${file.fileName}", e) - } - } - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt deleted file mode 100644 index 87970b90..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/FileStreamData.kt +++ /dev/null @@ -1,10 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import kotlinx.coroutines.sync.Semaphore -import java.nio.channels.FileChannel - -data class FileStreamData( - var fileStream: FileChannel?, - val fileLock: Semaphore, - var chunksToDownload: Int, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt deleted file mode 100644 index 76929969..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/GlobalDownloadCounter.kt +++ /dev/null @@ -1,6 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -data class GlobalDownloadCounter( - var totalBytesCompressed: Long = 0, - var totalBytesUncompressed: Long = 0, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt deleted file mode 100644 index 05373c88..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/IManifestProvider.kt +++ /dev/null @@ -1,36 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -/** - * An interface for persisting depot manifests for Steam content downloading - * - * @author Oxters - * @since 2024-11-06 - */ -interface IManifestProvider { - - /** - * Ask a provider to fetch a specific depot manifest - * @return A [Pair] object with a [DepotManifest] and its checksum if it exists otherwise null - */ - fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? - - /** - * Ask a provider to fetch the most recent manifest used of a depot - * @return A [Pair] object with a [DepotManifest] and its checksum if it exists otherwise null - */ - fun fetchLatestManifest(depotID: Int): DepotManifest? - - /** - * Ask a provider to set the most recent manifest ID used of a depot - */ - fun setLatestManifestId(depotID: Int, manifestID: Long) - - /** - * Update the persistent depot manifest - * @param manifest The depot manifest - * @return The checksum of the depot manifest - */ - fun updateManifest(manifest: DepotManifest) -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt deleted file mode 100644 index 8f680953..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/MemoryManifestProvider.kt +++ /dev/null @@ -1,28 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -import `in`.dragonbra.javasteam.types.DepotManifest - -/** - * @author Oxters - * @since 2024-11-06 - */ -class MemoryManifestProvider : IManifestProvider { - - private val depotManifests = mutableMapOf>() - - private val latestManifests = mutableMapOf() - - override fun fetchManifest(depotID: Int, manifestID: Long): DepotManifest? = - depotManifests[depotID]?.get(manifestID) - - override fun fetchLatestManifest(depotID: Int): DepotManifest? = - latestManifests[depotID]?.let { fetchManifest(depotID, it) } - - override fun setLatestManifestId(depotID: Int, manifestID: Long) { - latestManifests[depotID] = manifestID - } - - override fun updateManifest(manifest: DepotManifest) { - depotManifests.getOrPut(manifest.depotID) { mutableMapOf() }[manifest.manifestGID] = manifest - } -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt b/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt deleted file mode 100644 index afe5adc3..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/contentdownloader/ProgressCallback.kt +++ /dev/null @@ -1,8 +0,0 @@ -package `in`.dragonbra.javasteam.steam.contentdownloader - -/** - * Interface for Java to implement for progress updates - */ -fun interface ProgressCallback { - fun onProgress(progress: Float) -} diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt similarity index 74% rename from src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt rename to src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt index 807670f5..1d9068aa 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/AuthToken.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/CDNAuthToken.kt @@ -1,14 +1,14 @@ -package `in`.dragonbra.javasteam.steam.cdn +package `in`.dragonbra.javasteam.steam.handlers.steamcontent import `in`.dragonbra.javasteam.enums.EResult -import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetCDNAuthToken_Response +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.callback.ServiceMethodResponse import java.util.Date /** * This is received when a CDN auth token is received */ -class AuthToken(message: ServiceMethodResponse) { +class CDNAuthToken(message: ServiceMethodResponse) { /** * Result of the operation diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt index 335530c3..5e66c4e6 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt @@ -5,9 +5,9 @@ import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystem import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetManifestRequestCode_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetServersForSteamPipe_Request import `in`.dragonbra.javasteam.rpc.service.ContentServerDirectory -import `in`.dragonbra.javasteam.steam.cdn.AuthToken import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler +import `in`.dragonbra.javasteam.steam.handlers.steamcontent.CDNAuthToken import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages import `in`.dragonbra.javasteam.steam.webapi.ContentServerDirectoryService import kotlinx.coroutines.CoroutineScope @@ -65,7 +65,7 @@ class SteamContent : ClientMsgHandler() { branch: String? = null, branchPasswordHash: String? = null, parentScope: CoroutineScope, - ): Deferred = parentScope.async { + ): Deferred = parentScope.async { var localBranch = branch var localBranchPasswordHash = branchPasswordHash @@ -89,24 +89,24 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getManifestRequestCode(request).await() val response = message.body.build() - return@async response.manifestRequestCode.toULong() + return@async response.manifestRequestCode } /** * Request product information for an app or package - * Results are returned in a [AuthToken]. + * Results are returned in a [CDNAuthToken]. * * @param app App id requested. * @param depot Depot id requested. * @param hostName CDN host name being requested. - * @return The [AuthToken] containing the result. + * @return The [CDNAuthToken] containing the result. */ fun getCDNAuthToken( app: Int, depot: Int, hostName: String, parentScope: CoroutineScope, - ): Deferred = parentScope.async { + ): Deferred = parentScope.async { val request = CContentServerDirectory_GetCDNAuthToken_Request.newBuilder().apply { this.appId = app this.depotId = depot @@ -115,7 +115,7 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getCDNAuthToken(request).await() - return@async AuthToken(message) + return@async CDNAuthToken(message) } /** diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt index 0bb0a2af..9c91907a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/ISteamConfigurationBuilder.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import okhttp3.OkHttpClient import java.util.* @@ -96,14 +95,6 @@ interface ISteamConfigurationBuilder { */ fun withServerListProvider(provider: IServerListProvider): ISteamConfigurationBuilder - /** - * Configures the depot manifest provider for this [SteamConfiguration]. - * - * @param provider The depot manifest provider to use. - * @return A builder with modified configuration. - */ - fun withManifestProvider(provider: IManifestProvider): ISteamConfigurationBuilder - /** * Configures the Universe that this [SteamConfiguration] belongs to. * diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt index 13212928..44a7c41a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfiguration.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import `in`.dragonbra.javasteam.steam.discovery.SmartCMServerList import `in`.dragonbra.javasteam.steam.steamclient.SteamClient @@ -79,12 +78,6 @@ class SteamConfiguration internal constructor(private val state: SteamConfigurat val serverListProvider: IServerListProvider get() = state.serverListProvider - /** - * The depot manifest provider to use. - */ - val depotManifestProvider: IManifestProvider - get() = state.depotManifestProvider - /** * The Universe to connect to. This should always be [EUniverse.Public] unless you work at Valve and are using this internally. If this is you, hello there. */ diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt index 6ce754d2..c275477a 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationBuilder.kt @@ -4,8 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider -import `in`.dragonbra.javasteam.steam.contentdownloader.MemoryManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import `in`.dragonbra.javasteam.steam.discovery.MemoryServerListProvider import `in`.dragonbra.javasteam.steam.webapi.WebAPI @@ -72,11 +70,6 @@ class SteamConfigurationBuilder : ISteamConfigurationBuilder { return this } - override fun withManifestProvider(provider: IManifestProvider): ISteamConfigurationBuilder { - state.depotManifestProvider = provider - return this - } - override fun withUniverse(universe: EUniverse): ISteamConfigurationBuilder { state.universe = universe return this @@ -108,7 +101,6 @@ class SteamConfigurationBuilder : ISteamConfigurationBuilder { httpClient = OkHttpClient(), protocolTypes = EnumSet.of(ProtocolTypes.TCP, ProtocolTypes.WEB_SOCKET), serverListProvider = MemoryServerListProvider(), - depotManifestProvider = MemoryManifestProvider(), universe = EUniverse.Public, webAPIBaseAddress = WebAPI.DEFAULT_BASE_ADDRESS, cellID = 0, diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt index fbce399a..00bf1ff4 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/configuration/SteamConfigurationState.kt @@ -4,7 +4,6 @@ import `in`.dragonbra.javasteam.enums.EClientPersonaStateFlag import `in`.dragonbra.javasteam.enums.EUniverse import `in`.dragonbra.javasteam.networking.steam3.IConnectionFactory import `in`.dragonbra.javasteam.networking.steam3.ProtocolTypes -import `in`.dragonbra.javasteam.steam.contentdownloader.IManifestProvider import `in`.dragonbra.javasteam.steam.discovery.IServerListProvider import okhttp3.OkHttpClient import java.util.* @@ -22,7 +21,6 @@ data class SteamConfigurationState( var httpClient: OkHttpClient, var protocolTypes: EnumSet, var serverListProvider: IServerListProvider, - var depotManifestProvider: IManifestProvider, var universe: EUniverse, var webAPIBaseAddress: String, var webAPIKey: String?, diff --git a/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt b/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt index 54565d32..d30c5548 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/webapi/ContentServerDirectoryService.kt @@ -8,6 +8,7 @@ import `in`.dragonbra.javasteam.steam.cdn.Server */ object ContentServerDirectoryService { + @JvmStatic internal fun convertServerList( response: CContentServerDirectory_GetServersForSteamPipe_Response, ): List = response.serversList.map { child -> @@ -18,21 +19,21 @@ object ContentServerDirectoryService { Server.ConnectionProtocol.HTTP } - Server( - protocol = protocol, - host = child.host, - vHost = child.vhost, - port = if (protocol == Server.ConnectionProtocol.HTTPS) 443 else 80, - type = child.type, - sourceID = child.sourceId, - cellID = child.cellId, - load = child.load, - weightedLoad = child.weightedLoad, - numEntries = child.numEntriesInClientList, - steamChinaOnly = child.steamChinaOnly, - useAsProxy = child.useAsProxy, - proxyRequestPathTemplate = child.proxyRequestPathTemplate, - allowedAppIds = child.allowedAppIdsList.toIntArray() - ) + Server().apply { + this.protocol = protocol + this.host = child.host + this.vHost = child.vhost + this.port = if (protocol == Server.ConnectionProtocol.HTTPS) 443 else 80 + this.type = child.type + this.sourceId = child.sourceId + this.cellId = child.cellId + this.load = child.load + this.weightedLoad = child.weightedLoad + this.numEntries = child.numEntriesInClientList + this.steamChinaOnly = child.steamChinaOnly + this.useAsProxy = child.useAsProxy + this.proxyRequestPathTemplate = child.proxyRequestPathTemplate + this.allowedAppIds = child.allowedAppIdsList.toIntArray() + } } } diff --git a/src/main/java/in/dragonbra/javasteam/types/PubFile.kt b/src/main/java/in/dragonbra/javasteam/types/PubFile.kt new file mode 100644 index 00000000..fe50ce11 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/types/PubFile.kt @@ -0,0 +1,108 @@ +package `in`.dragonbra.javasteam.types + +/** + * The base class used for wrapping common ULong types, to introduce type safety and distinguish between common types. + */ +@Suppress("unused") +abstract class UInt64Handle : Any { + + /** + * Gets or sets the value. + */ + protected open var value: ULong = 0UL + + /** + * @constructor Initializes a new instance of the [UInt64Handle] class. + */ + protected constructor() + + /** + * Initializes a new instance of the [UInt64Handle] class. + * @param value The value to initialize this handle to. + */ + protected constructor(value: ULong) { + this.value = value + } + + /** + * Returns a hash code for this instance. + * @return A hash code for this instance, suitable for use in hashing algorithms and data structures like a hash table. + */ + override fun hashCode(): Int = value.hashCode() + + /** + * Determines whether the specified object is equal to this instance. + * @param other The object to compare with this instance. + * @return true if the specified object is equal to this instance; otherwise, false. + */ + override fun equals(other: Any?): Boolean { + if (other is UInt64Handle) { + return other.value == value + } + + return false + } + + /** + * Returns a string that represents this instance. + * @return A string that represents this instance. + */ + override fun toString(): String = value.toString() + + /** + * TODO + */ + fun toLong(): Long = value.toLong() + + /** + * Indicates whether the current object is equal to another object of the same type. + * @param other An object to compare with this object. + * @return true if the current object is equal to the other parameter; otherwise, false. + */ + fun equals(other: UInt64Handle?): Boolean { + if (other == null) { + return false + } + return value == other.value + } +} + +/** + * Represents a handle to a published file on the Steam workshop. + */ +class PublishedFileID : UInt64Handle { + + /** + * Initializes a new instance of the PublishedFileID class. + * @param fileId The file id. + */ + constructor(fileId: Long = Long.MAX_VALUE) : super(fileId.toULong()) + + companion object { + /** + * Implements the operator ==. + * @param a The first published file. + * @param b The second published file. + * @return The result of the operator. + */ + fun equals(a: PublishedFileID?, b: PublishedFileID?): Boolean { + if (a === b) { + return true + } + + if (a == null || b == null) { + return false + } + + return a.value == b.value + } + + /** + * Implements the operator !=. + * @param a The first published file. + * @param b The second published file. + * @return The result of the operator. + */ + fun notEquals(a: PublishedFileID?, b: PublishedFileID?): Boolean = !equals(a, b) + } +} diff --git a/src/main/java/in/dragonbra/javasteam/util/Utils.java b/src/main/java/in/dragonbra/javasteam/util/Utils.java index 17c122b3..c027c605 100644 --- a/src/main/java/in/dragonbra/javasteam/util/Utils.java +++ b/src/main/java/in/dragonbra/javasteam/util/Utils.java @@ -204,40 +204,4 @@ public static long crc32(byte[] bytes, int offset, int length) { checksum.update(bytes, offset, length); return checksum.getValue(); } - - /** - * Validate a file against Steam3 Chunk data - * - * @param fs FileInputStream to read from - * @param chunkData Array of ChunkData to validate against - * @return List of ChunkData that are needed - * @throws IOException If there's an error reading the file - * @throws ClosedChannelException If this channel is closed - * @throws IllegalArgumentException If the new position is negative - */ - @SuppressWarnings("resource") - public static List validateSteam3FileChecksums(RandomAccessFile fs, ChunkData[] chunkData) throws IOException { - List neededChunks = new ArrayList<>(); - int read; - - for (ChunkData data : chunkData) { - byte[] chunk = new byte[data.getUncompressedLength()]; - fs.getChannel().position(data.getOffset()); - read = fs.read(chunk, 0, data.getUncompressedLength()); - - byte[] tempChunk; - if (read > 0 && read < data.getUncompressedLength()) { - tempChunk = Arrays.copyOf(chunk, read); - } else { - tempChunk = chunk; - } - - int adler = Adler32.calculate(tempChunk); - if (adler != data.getChecksum()) { - neededChunks.add(data); - } - } - - return neededChunks; - } } diff --git a/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java b/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java index 30e9bd95..2bf4ef4a 100644 --- a/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java +++ b/src/test/java/in/dragonbra/javasteam/steam/cdn/CDNClientTest.java @@ -42,7 +42,7 @@ public void throwsSteamKitWebExceptionOnUnsuccessfulWebResponseForManifest() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); // JVM will throw ExecutionException Exception exception = Assertions.assertThrows(ExecutionException.class, () -> @@ -64,7 +64,7 @@ public void throwsSteamKitWebExceptionOnUnsuccessfulWebResponseForChunk() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0L, 0, 0); // JVM will throw ExecutionException @@ -87,7 +87,7 @@ public void throwsWhenNoChunkIDIsSet() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(); // JVM will throw ExecutionException @@ -111,7 +111,7 @@ public void throwsWhenDestinationBufferSmaller() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0, 32, 64); // JVM will throw ExecutionException @@ -135,7 +135,7 @@ public void throwsWhenDestinationBufferSmallerWithDepotKey() { var configuration = SteamConfiguration.create(x -> x.withHttpClient(httpClient)); var steam = new SteamClient(configuration); try (var client = new Client(steam)) { - var server = new Server(Server.ConnectionProtocol.HTTP, "localhost", "localhost", 80); + var server = Server.fromHostAndPort("localhost", 80); var chunk = new ChunkData(new byte[]{(byte) 0xFF}, 0, 0, 32, 64); // JVM will throw ExecutionException From 873266a69d84457e6a7bb0feb8b1e327807cc67c Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 11:14:13 -0500 Subject: [PATCH 02/44] Clean up dependencies --- gradle/libs.versions.toml | 8 ++++++++ javasteam-depotdownloader/build.gradle.kts | 7 +++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index edaaa11a..24ebc318 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -21,6 +21,10 @@ publishPlugin = "2.0.0" # https://mvnrepository.com/artifact/io.github.gradle-ne xz = "1.11" # https://mvnrepository.com/artifact/org.tukaani/xz zstd = "1.5.7-6" # https://search.maven.org/artifact/com.github.luben/zstd-jni +# Depot Downloader +kotlin-serialization-json = "1.9.0" # https://mvnrepository.com/artifact/org.jetbrains.kotlinx/kotlinx-serialization-json-jvm +okio = "3.16.0" # https://mvnrepository.com/artifact/com.squareup.okio/okio + # Testing Lib versions commons-io = "2.21.0" # https://mvnrepository.com/artifact/commons-io/commons-io commonsCodec = "1.20.0" # https://mvnrepository.com/artifact/commons-codec/commons-codec @@ -47,6 +51,10 @@ protobuf-protoc = { module = "com.google.protobuf:protoc", version.ref = "protob xz = { module = "org.tukaani:xz", version.ref = "xz" } zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } +# Depot Downloader +kotlin-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlin-serialization-json"} +okio = { module = "com.squareup.okio:okio", version.ref = "okio"} + # Tests test-commons-codec = { module = "commons-codec:commons-codec", version.ref = "commonsCodec" } test-commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" } diff --git a/javasteam-depotdownloader/build.gradle.kts b/javasteam-depotdownloader/build.gradle.kts index 5ad11c36..df2c31b1 100644 --- a/javasteam-depotdownloader/build.gradle.kts +++ b/javasteam-depotdownloader/build.gradle.kts @@ -67,14 +67,13 @@ tasks.withType { } dependencies { - implementation(rootProject) // TODO verify if this causes something like a circular dependency. - - implementation("com.squareup.okio:okio:3.16.0") - implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.9.0") + implementation(rootProject) implementation(libs.bundles.ktor) implementation(libs.commons.lang3) implementation(libs.kotlin.coroutines) + implementation(libs.kotlin.serialization.json) implementation(libs.kotlin.stdib) + implementation(libs.okio) implementation(libs.protobuf.java) } From 1918b7c40d17bac34efa0fef31aef5e624697d25 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 13:03:49 -0500 Subject: [PATCH 03/44] Finalize some methods, and some cleanup. --- .../depotdownloader/CDNClientPool.kt | 19 +---- .../depotdownloader/ContentDownloader.kt | 82 +++++++++++-------- .../depotdownloader/DepotConfigStore.kt | 4 +- .../javasteam/depotdownloader/HttpClient.kt | 42 +++++----- .../javasteam/depotdownloader/Util.kt | 22 ++++- 5 files changed, 94 insertions(+), 75 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 6ea50137..81f341aa 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -7,9 +7,6 @@ import `in`.dragonbra.javasteam.steam.steamclient.SteamClient import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.cancel import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlin.jvm.Throws @@ -18,7 +15,7 @@ import kotlin.jvm.Throws * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. * @param steamClient an instance of [SteamClient] * @param appId the selected app id to ensure an endpoint supports the download. - * @param scope (optional) the [CoroutineScope] to use. + * @param scope the [CoroutineScope] to use. * @param debug enable or disable logging through [LogManager] * * @author Oxters @@ -28,18 +25,10 @@ import kotlin.jvm.Throws class CDNClientPool( private val steamClient: SteamClient, private val appId: Int, - private val scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.IO), + private val scope: CoroutineScope, debug: Boolean = false, ) : AutoCloseable { - companion object { - fun init( - steamClient: SteamClient, - appId: Int, - debug: Boolean, - ): CDNClientPool = CDNClientPool(steamClient = steamClient, appId = appId, debug = debug) - } - private var logger: Logger? = null private val servers: ArrayList = arrayListOf() @@ -63,8 +52,6 @@ class CDNClientPool( } override fun close() { - scope.cancel() - servers.clear() cdnClient = null @@ -134,7 +121,7 @@ class CDNClientPool( if (servers[nextServer % servers.count()] == server) { nextServer++ - // TODO: Add server to ContentServerPenalty + // TODO: (SK) Add server to ContentServerPenalty } } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt index 2571e2c0..0f85c9a6 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -112,10 +112,10 @@ class ContentDownloader @JvmOverloads constructor( // What is a PriorityQueue? private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private val items = CopyOnWriteArrayList(ArrayList()) private val listeners = CopyOnWriteArrayList() - private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private var logger: Logger? = null private val isStarted: AtomicBoolean = AtomicBoolean(false) private val processingChannel = Channel(Channel.UNLIMITED) @@ -242,7 +242,7 @@ class ContentDownloader @JvmOverloads constructor( filesystem.createDirectories(fileFinalPath.parent!!) filesystem.createDirectories(fileStagingPath.parent!!) - HttpClient.httpClient.use { client -> + HttpClient.getClient(maxDownloads).use { client -> logger?.debug("Starting download of $fileName...") val response = client.get(url) @@ -300,7 +300,7 @@ class ContentDownloader @JvmOverloads constructor( var depotManifestIds = depotManifestIds.toMutableList() val steamUser = requireNotNull(steam3!!.steamUser) - cdnClientPool = CDNClientPool.init(steamClient, appId, debug) + cdnClientPool = CDNClientPool(steamClient, appId, scope, debug) // Load our configuration data containing the depots currently installed var configPath = config.installPath @@ -1578,9 +1578,12 @@ class ContentDownloader @JvmOverloads constructor( return } - items.add(0, item) - - notifyListeners { it.onItemAdded(item, 0) } + try { + items.add(0, item) + notifyListeners { it.onItemAdded(item, 0) } + } catch (e: Exception) { + logger?.error(e) + } } fun addAt(index: Int, item: DownloadItem): Boolean { @@ -1604,11 +1607,16 @@ class ContentDownloader @JvmOverloads constructor( return null } - return if (items.isNotEmpty()) { - val item = items.removeAt(0) - notifyListeners { it.onItemRemoved(item, 0) } - item - } else { + return try { + if (items.isNotEmpty()) { + val item = items.removeAt(0) + notifyListeners { it.onItemRemoved(item, 0) } + item + } else { + null + } + } catch (e: IndexOutOfBoundsException) { + logger?.error(e) null } } @@ -1619,12 +1627,17 @@ class ContentDownloader @JvmOverloads constructor( return null } - return if (items.isNotEmpty()) { - val lastIndex = items.size - 1 - val item = items.removeAt(lastIndex) - notifyListeners { it.onItemRemoved(item, lastIndex) } - item - } else { + return try { + if (items.isNotEmpty()) { + val lastIndex = items.size - 1 + val item = items.removeAt(lastIndex) + notifyListeners { it.onItemRemoved(item, lastIndex) } + item + } else { + null + } + } catch (e: IndexOutOfBoundsException) { + logger?.error(e) null } } @@ -1636,11 +1649,16 @@ class ContentDownloader @JvmOverloads constructor( } val index = items.indexOf(item) - return if (index >= 0) { - items.removeAt(index) - notifyListeners { it.onItemRemoved(item, index) } - true - } else { + return try { + if (index >= 0) { + items.removeAt(index) + notifyListeners { it.onItemRemoved(item, index) } + true + } else { + false + } + } catch (e: IndexOutOfBoundsException) { + logger?.error(e) false } } @@ -1720,6 +1738,16 @@ class ContentDownloader @JvmOverloads constructor( remainingItems.set(initialItems.size) initialItems.forEach { processingChannel.send(it) } + if (ClientLancache.useLanCacheServer) { + logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") + + // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely + // be served much faster than over the internet. Steam internally has this behavior as well. + if (maxDownloads == 8) { + maxDownloads = 25 + } + } + repeat(remainingItems.get()) { // Process exactly this many ensureActive() @@ -1733,16 +1761,6 @@ class ContentDownloader @JvmOverloads constructor( ClientLancache.detectLancacheServer() } - if (ClientLancache.useLanCacheServer) { - logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") - - // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely - // be served much faster than over the internet. Steam internally has this behavior as well. - if (maxDownloads == 8) { - maxDownloads = 25 - } - } - // Set some configuration values, first. config = config.copy( downloadManifestOnly = item.downloadManifestOnly, diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index 2f746eb2..56dea76e 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -20,9 +20,6 @@ data class DepotConfigStore( private val json = Json { prettyPrint = true } - val isLoaded: Boolean - get() = instance != null - fun loadFromFile(path: Path) { // require(!isLoaded) { "Config already loaded" } @@ -48,6 +45,7 @@ data class DepotConfigStore( } } + @Throws(IllegalArgumentException::class) fun getInstance(): DepotConfigStore = requireNotNull(instance) { "Config not loaded" } } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 03908dc2..3d626d55 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -13,34 +13,34 @@ import kotlinx.coroutines.isActive */ object HttpClient { - private var _httpClient: HttpClient? = null + private var httpClient: HttpClient? = null - val httpClient: HttpClient - get() { - if (_httpClient?.isActive != true) { - _httpClient = HttpClient(CIO) { - install(UserAgent) { - agent = "DepotDownloader/${Versions.getVersion()}" - } - engine { - maxConnectionsCount = 10 - endpoint { - maxConnectionsPerRoute = 5 - pipelineMaxSize = 20 - keepAliveTime = 5000 - connectTimeout = 5000 - requestTimeout = 30000 - } + fun getClient(maxConnections: Int = 8): HttpClient { + if (httpClient?.isActive != true) { + httpClient = HttpClient(CIO) { + install(UserAgent) { + agent = "DepotDownloader/${Versions.getVersion()}" + } + engine { + maxConnectionsCount = maxConnections + endpoint { + maxConnectionsPerRoute = (maxConnections / 2).coerceAtLeast(1) + pipelineMaxSize = maxConnections * 2 + keepAliveTime = 5000 + connectTimeout = 5000 + requestTimeout = 30000 } } } - return _httpClient!! } + return httpClient!! + } + fun close() { - if (httpClient.isActive) { - _httpClient?.close() - _httpClient = null + if (httpClient?.isActive == true) { + httpClient?.close() + httpClient = null } } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt index b69a0047..8f178765 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -5,6 +5,8 @@ import `in`.dragonbra.javasteam.enums.EDepotFileFlag import `in`.dragonbra.javasteam.types.ChunkData import `in`.dragonbra.javasteam.types.DepotManifest import `in`.dragonbra.javasteam.util.Adler32 +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger import okio.FileHandle import okio.FileSystem import okio.Path @@ -13,6 +15,7 @@ import okio.buffer import org.apache.commons.lang3.SystemUtils import java.io.IOException import java.security.MessageDigest +import java.security.NoSuchAlgorithmException /** * @author Lossy @@ -20,6 +23,8 @@ import java.security.MessageDigest */ object Util { + private val logger: Logger = LogManager.getLogger() + @JvmOverloads @JvmStatic fun getSteamOS(androidEmulation: Boolean = false): String { @@ -47,7 +52,12 @@ object Util { @JvmStatic fun getSteamArch(): String { - val arch = System.getProperty("os.arch")?.lowercase() ?: "" + val arch = try { + System.getProperty("os.arch")?.lowercase() ?: "" + } catch (e: Exception) { + logger.error(e) + "" + } return when { arch.contains("64") -> "64" arch.contains("86") -> "32" @@ -60,6 +70,7 @@ object Util { } @JvmStatic + @Throws(IOException::class) fun saveManifestToFile(directory: Path, manifest: DepotManifest): Boolean = try { val filename = directory / "${manifest.depotID}_${manifest.manifestGID}.manifest" manifest.saveToFile(filename.toString()) @@ -70,11 +81,13 @@ object Util { } true - } catch (e: Exception) { + } catch (e: IOException) { + logger.error(e) false } @JvmStatic + @Throws(NoSuchAlgorithmException::class, IllegalArgumentException::class, IOException::class) fun loadManifestFromFile( directory: Path, depotId: Int, @@ -98,7 +111,7 @@ object Util { if (expectedChecksum != null && expectedChecksum.contentEquals(currentChecksum)) { return DepotManifest.loadFromFile(filename.toString()) } else if (badHashWarning) { - println("Manifest $manifestId on disk did not match the expected checksum.") + logger.debug("Manifest $manifestId on disk did not match the expected checksum.") } } @@ -106,6 +119,7 @@ object Util { } @JvmStatic + @Throws(NoSuchAlgorithmException::class, IllegalArgumentException::class, IOException::class) fun fileSHAHash(filename: Path): ByteArray { val digest = MessageDigest.getInstance("SHA-1") @@ -131,6 +145,7 @@ object Util { * @return List of ChunkData that are needed * @throws IOException If there's an error reading the file */ + @JvmStatic @Throws(IOException::class) fun validateSteam3FileChecksums(handle: FileHandle, chunkData: List): List { val neededChunks = mutableListOf() @@ -155,6 +170,7 @@ object Util { } @JvmStatic + @Throws(IOException::class) fun dumpManifestToTextFile(depot: DepotDownloadInfo, manifest: DepotManifest) { val txtManifest = depot.installDir / "manifest_${depot.depotId}_${depot.manifestId}.txt" From e9a9079cfc51b8b668bb500074ddd8d3c63183ef Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 15:10:49 -0500 Subject: [PATCH 04/44] Fix nullable value in ConcurrentHashMap --- .../depotdownloader/ContentDownloader.kt | 4 ++-- .../javasteam/depotdownloader/Steam3Session.kt | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt index 0f85c9a6..09d201a0 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -696,7 +696,7 @@ class ContentDownloader @JvmOverloads constructor( return null } - val app = steam3!!.appInfo[appId] ?: return null + val app = steam3!!.appInfo[appId]?.value ?: return null val appInfo = app.keyValues val sectionKey = when (section) { @@ -729,7 +729,7 @@ class ContentDownloader @JvmOverloads constructor( steam3!!.requestPackageInfo(licenseQuery) licenseQuery.forEach { license -> - steam3!!.packageInfo[license]?.let { pkg -> + steam3!!.packageInfo[license]?.value?.let { pkg -> val appIds = pkg.keyValues["appids"].children.map { it.asInteger() } val depotIds = pkg.keyValues["depotids"].children.map { it.asInteger() } if (depotId in appIds) { diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 2b80f563..053e4928 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -42,8 +42,8 @@ class Steam3Session( internal val packageTokens = ConcurrentHashMap() internal val depotKeys = ConcurrentHashMap() internal val cdnAuthTokens = ConcurrentHashMap, CompletableDeferred>() - internal val appInfo = ConcurrentHashMap() - internal val packageInfo = ConcurrentHashMap() + internal val appInfo = ConcurrentHashMap>() + internal val packageInfo = ConcurrentHashMap>() internal val appBetaPasswords = ConcurrentHashMap() private var unifiedMessages: SteamUnifiedMessages? = null @@ -53,6 +53,9 @@ class Steam3Session( internal var steamCloud: SteamCloud? = null internal var steamPublishedFile: PublishedFile? = null + // ConcurrentHashMap can't have nullable Keys or Values + internal data class Optional(val value: T?) + init { if (debug) { logger = LogManager.getLogger(Steam3Session::class.java) @@ -127,10 +130,10 @@ class Steam3Session( appInfoMultiple.results.forEach { appInfo -> appInfo.apps.forEach { appValue -> val app = appValue.value - this.appInfo[app.id] = app + this.appInfo[app.id] = Optional(app) } appInfo.unknownApps.forEach { app -> - this.appInfo[app] = null + this.appInfo[app] = Optional(null) } } } @@ -166,10 +169,10 @@ class Steam3Session( packageInfoMultiple.results.forEach { pkgInfo -> pkgInfo.packages.forEach { pkgValue -> val pkg = pkgValue.value - packageInfo[pkg.id] = pkg + packageInfo[pkg.id] = Optional(pkg) } pkgInfo.unknownPackages.forEach { pkgValue -> - packageInfo[pkgValue] = null + packageInfo[pkgValue] = Optional(null) } } } From e3521aa8db3b172fe0f8bc06349b250521ce0a8b Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 21:16:38 -0500 Subject: [PATCH 05/44] Some tidying --- .../depotdownloader/CDNClientPool.kt | 19 ++++----- .../depotdownloader/ContentDownloader.kt | 2 +- .../depotdownloader/DepotConfigStore.kt | 2 - .../javasteam/depotdownloader/HttpClient.kt | 2 +- .../depotdownloader/Steam3Session.kt | 41 +++++++++---------- .../javasteam/depotdownloader/Util.kt | 1 + 6 files changed, 31 insertions(+), 36 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 81f341aa..33f0e264 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -2,18 +2,15 @@ package `in`.dragonbra.javasteam.depotdownloader import `in`.dragonbra.javasteam.steam.cdn.Client import `in`.dragonbra.javasteam.steam.cdn.Server -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.SteamContent -import `in`.dragonbra.javasteam.steam.steamclient.SteamClient import `in`.dragonbra.javasteam.util.log.LogManager import `in`.dragonbra.javasteam.util.log.Logger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlin.jvm.Throws /** * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. - * @param steamClient an instance of [SteamClient] + * @param steamSession an instance of [Steam3Session] * @param appId the selected app id to ensure an endpoint supports the download. * @param scope the [CoroutineScope] to use. * @param debug enable or disable logging through [LogManager] @@ -23,7 +20,7 @@ import kotlin.jvm.Throws * @since Nov 7, 2024 */ class CDNClientPool( - private val steamClient: SteamClient, + private val steamSession: Steam3Session, private val appId: Int, private val scope: CoroutineScope, debug: Boolean = false, @@ -44,7 +41,7 @@ class CDNClientPool( private set init { - cdnClient = Client(steamClient) + cdnClient = Client(steamSession.steamClient) if (debug) { logger = LogManager.getLogger(CDNClientPool::class.java) @@ -66,8 +63,8 @@ class CDNClientPool( servers.clear() } - val serversForSteamPipe = steamClient.getHandler()!!.getServersForSteamPipe( - cellId = steamClient.cellID ?: 0, + val serversForSteamPipe = steamSession.steamContent!!.getServersForSteamPipe( + cellId = steamSession.steamClient.cellID ?: 0, maxNumServers = maxNumServers, parentScope = scope ).await() @@ -85,8 +82,8 @@ class CDNClientPool( servers.addAll(weightedCdnServers) - // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } - logger?.debug("Found ${servers.size} Servers: \n") + // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } + logger?.debug("Found ${servers.size} Servers") if (servers.isEmpty()) { throw Exception("Failed to retrieve any download servers.") @@ -108,7 +105,7 @@ class CDNClientPool( logger?.debug("Returning connection: $server") - // nothing to do, maybe remove from ContentServerPenalty? + // (SK) nothing to do, maybe remove from ContentServerPenalty? } suspend fun returnBrokenConnection(server: Server?) = mutex.withLock { diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt index 09d201a0..c539e6ce 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt @@ -300,7 +300,7 @@ class ContentDownloader @JvmOverloads constructor( var depotManifestIds = depotManifestIds.toMutableList() val steamUser = requireNotNull(steam3!!.steamUser) - cdnClientPool = CDNClientPool(steamClient, appId, scope, debug) + cdnClientPool = CDNClientPool(steam3!!, appId, scope, debug) // Load our configuration data containing the depots currently installed var configPath = config.installPath diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index 56dea76e..b7297734 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -21,8 +21,6 @@ data class DepotConfigStore( private val json = Json { prettyPrint = true } fun loadFromFile(path: Path) { - // require(!isLoaded) { "Config already loaded" } - instance = if (FileSystem.SYSTEM.exists(path)) { FileSystem.SYSTEM.read(path) { json.decodeFromString(readUtf8()) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 3d626d55..340f85a9 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -19,7 +19,7 @@ object HttpClient { if (httpClient?.isActive != true) { httpClient = HttpClient(CIO) { install(UserAgent) { - agent = "DepotDownloader/${Versions.getVersion()}" + agent = "JavaSteam-DepotDownloader/${Versions.getVersion()}" } engine { maxConnectionsCount = maxConnections diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 053e4928..3f7e7099 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -32,12 +32,14 @@ import java.util.concurrent.ConcurrentHashMap * @since Oct 1, 2025 */ class Steam3Session( - private val steamClient: SteamClient, + internal val steamClient: SteamClient, debug: Boolean, ) : Closeable { private var logger: Logger? = null + private var isAborted: Boolean = false // Stubbed, no way to set true/false yet. + internal val appTokens = ConcurrentHashMap() internal val packageTokens = ConcurrentHashMap() internal val depotKeys = ConcurrentHashMap() @@ -96,7 +98,7 @@ class Steam3Session( } suspend fun requestAppInfo(appId: Int, bForce: Boolean = false) { - if (appInfo.containsKey(appId) && !bForce) { + if ((appInfo.containsKey(appId) && !bForce) || isAborted) { return } @@ -145,7 +147,9 @@ class Steam3Session( // I have a silly race condition??? val packages = packageIds.filter { !packageInfo.containsKey(it) } - if (packages.isEmpty()) return + if (packages.isEmpty() || isAborted) { + return + } val packageRequests = arrayListOf() @@ -192,7 +196,7 @@ class Steam3Session( } suspend fun requestDepotKey(depotId: Int, appId: Int = 0) { - if (depotKeys.containsKey(depotId)) { + if (depotKeys.containsKey(depotId) || isAborted) { return } @@ -216,6 +220,10 @@ class Steam3Session( manifestId: Long, branch: String, ): ULong = withContext(Dispatchers.IO) { + if (isAborted) { + return@withContext 0UL + } + val requestCode = steamContent!!.getManifestRequestCode( depotId = depotId, appId = appId, @@ -229,7 +237,7 @@ class Steam3Session( logger?.error("No manifest request code was returned for depot $depotId from app $appId, manifest $manifestId") if (steamClient.isDisconnected) { - logger?.debug("Suggestion: Try logging in with -username as old manifests may not be available for anonymous accounts.") + logger?.debug("Suggestion: Try logging in with a username as old manifests may not be available for anonymous accounts.") } } else { logger?.debug("Got manifest request code for depot $depotId from app $appId, manifest $manifestId, result: $requestCode") @@ -252,30 +260,21 @@ class Steam3Session( val completion = CompletableDeferred() - val existing = cdnAuthTokens.putIfAbsent(cdnKey, completion) - if (existing != null) { + if (isAborted || cdnAuthTokens.putIfAbsent(cdnKey, completion) != null) { return@withContext } logger?.debug("Requesting CDN auth token for ${server.host}") - try { - val cdnAuth = steamContent!!.getCDNAuthToken(appId, depotId, server.host!!, this).await() - - logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") + val cdnAuth = steamContent!!.getCDNAuthToken(appId, depotId, server.host!!, this).await() - if (cdnAuth.result != EResult.OK) { - cdnAuthTokens.remove(cdnKey) // Remove failed promise - completion.completeExceptionally(Exception("Failed to get CDN auth token: ${cdnAuth.result}")) - return@withContext - } + logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") - completion.complete(cdnAuth) - } catch (e: Exception) { - logger?.error(e) - cdnAuthTokens.remove(cdnKey) // Remove failed promise - completion.completeExceptionally(e) + if (cdnAuth.result != EResult.OK) { + return@withContext } + + completion.complete(cdnAuth) } suspend fun checkAppBetaPassword(appId: Int, password: String) { diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt index 8f178765..9d2bc32f 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -103,6 +103,7 @@ object Util { readByteArray() } } catch (e: IOException) { + logger.error(e) null } From b0dc78b13d649945963de1b0065643872ddbae6b Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 3 Oct 2025 21:55:34 -0500 Subject: [PATCH 06/44] Add maven and update workflows --- .github/workflows/javasteam-build-push.yml | 4 +++ javasteam-depotdownloader/build.gradle.kts | 39 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/javasteam-build-push.yml b/.github/workflows/javasteam-build-push.yml index 70c27cb9..cbc6ad95 100644 --- a/.github/workflows/javasteam-build-push.yml +++ b/.github/workflows/javasteam-build-push.yml @@ -51,3 +51,7 @@ jobs: with: name: javasteam-tf path: javasteam-tf/build/libs + - uses: actions/upload-artifact@v4 + with: + name: javasteam-depotdownloader + path: javasteam-depotdownloader/build/libs diff --git a/javasteam-depotdownloader/build.gradle.kts b/javasteam-depotdownloader/build.gradle.kts index df2c31b1..e22ddc43 100644 --- a/javasteam-depotdownloader/build.gradle.kts +++ b/javasteam-depotdownloader/build.gradle.kts @@ -28,7 +28,6 @@ protobuf.protoc { artifact = libs.protobuf.protoc.get().toString() } - java { sourceCompatibility = JavaVersion.toVersion(libs.versions.java.get()) targetCompatibility = JavaVersion.toVersion(libs.versions.java.get()) @@ -77,3 +76,41 @@ dependencies { implementation(libs.okio) implementation(libs.protobuf.java) } + +/* Artifact publishing */ +publishing { + publications { + create("mavenJava") { + from(components["java"]) + pom { + name = "JavaSteam-depotdownloader" + packaging = "jar" + description = "Depot Downloader for JavaSteam." + url = "https://github.com/Longi94/JavaSteam" + inceptionYear = "2025" + scm { + connection = "scm:git:git://github.com/Longi94/JavaSteam.git" + developerConnection = "scm:git:ssh://github.com:Longi94/JavaSteam.git" + url = "https://github.com/Longi94/JavaSteam/tree/master" + } + licenses { + license { + name = "MIT License" + url = "https://www.opensource.org/licenses/mit-license.php" + } + } + developers { + developer { + id = "Longi" + name = "Long Tran" + email = "lngtrn94@gmail.com" + } + } + } + } + } +} + +signing { + sign(publishing.publications["mavenJava"]) +} From dc68c2230c99e9d50b0c27b5fdd6b4eaaae57b1d Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 5 Oct 2025 01:52:09 -0500 Subject: [PATCH 07/44] Rename ContentDownloader and ContentDownloaderException to DepotDownloader and DepotDownloaderException. Add kDoc. Change DepotDownloader to use a fifo based system to download content in order of add. --- .../depotdownloader/CDNClientPool.kt | 14 +- .../ContentDownloaderException.kt | 7 - .../depotdownloader/DepotConfigStore.kt | 7 + ...ontentDownloader.kt => DepotDownloader.kt} | 489 ++++++------------ .../DepotDownloaderException.kt | 11 + .../javasteam/depotdownloader/HttpClient.kt | 5 + .../depotdownloader/IDownloadListener.kt | 78 ++- .../depotdownloader/Steam3Session.kt | 14 +- .../depotdownloader/data/ChunkMatch.kt | 7 + .../depotdownloader/data/DepotDownloadInfo.kt | 12 + .../depotdownloader/data/DepotFilesData.kt | 12 + .../depotdownloader/data/DownloadCounters.kt | 17 + .../depotdownloader/data/DownloadItems.kt | 34 +- .../depotdownloader/data/DownloadProgress.kt | 45 +- .../depotdownloader/data/FileStreamData.kt | 8 + .../_023_downloadapp/SampleDownloadApp.java | 123 ++--- 16 files changed, 443 insertions(+), 440 deletions(-) delete mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt rename javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/{ContentDownloader.kt => DepotDownloader.kt} (82%) create mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 33f0e264..58ac5366 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -9,11 +9,15 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock /** - * [CDNClientPool] provides a pool of connections to CDN endpoints, requesting CDN tokens as needed. - * @param steamSession an instance of [Steam3Session] - * @param appId the selected app id to ensure an endpoint supports the download. - * @param scope the [CoroutineScope] to use. - * @param debug enable or disable logging through [LogManager] + * Manages a pool of CDN server connections for efficient content downloading. + * This class maintains a list of available CDN servers, automatically selects appropriate + * servers based on load and app compatibility, and handles connection rotation when + * servers fail or become unavailable. + * + * @param steamSession The Steam3 session for server communication + * @param appId The application ID to download - used to filter compatible CDN servers + * @param scope The coroutine scope for async operations + * @param debug If true, enables debug logging * * @author Oxters * @author Lossy diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt deleted file mode 100644 index 5a514926..00000000 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloaderException.kt +++ /dev/null @@ -1,7 +0,0 @@ -package `in`.dragonbra.javasteam.depotdownloader - -/** - * @author Lossy - * @since Oct 1, 2025 - */ -class ContentDownloaderException(value: String) : Exception(value) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index b7297734..0bfebe14 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -6,6 +6,13 @@ import okio.FileSystem import okio.Path /** + * Singleton storage for tracking installed depot manifests. + * Persists manifest IDs to disk to enable incremental updates and avoid + * re-downloading unchanged content. The configuration is serialized as JSON + * and must be loaded via [loadFromFile] before use. + * + * @property installedManifestIDs Map of depot IDs to their currently installed manifest IDs + * * @author Lossy * @since Oct 1, 2025 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt similarity index 82% rename from javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt rename to javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index c539e6ce..481a6a6a 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/ContentDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -52,7 +52,6 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay import kotlinx.coroutines.ensureActive -import kotlinx.coroutines.future.future import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex @@ -70,31 +69,48 @@ import java.lang.IllegalStateException import java.time.Instant import java.time.temporal.ChronoUnit import java.util.concurrent.* -import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger import kotlin.collections.mutableListOf import kotlin.collections.set import kotlin.text.toLongOrNull /** - * [ContentDownloader] is a JavaSteam module that is able to download Games, Workshop Items, and other content from Steam. - * @param steamClient an instance of [SteamClient] - * @param licenses a list of licenses the logged-in user has. This is provided by [LicenseListCallback] - * @param debug enable or disable logging through [LogManager] - * @param useLanCache try and detect a local Steam Cache server. - * @param maxDownloads the number of simultaneous downloads. + * Downloads games, workshop items, and other Steam content via depot manifests. + * + * This class provides a queue-based FIFO download system that processes items sequentially. + * Items are added via [add] or [addAll] and processed automatically in order. The processing + * loop starts immediately upon construction and waits for items to be queued. + * + * ## Download Process + * 1. Validates account access and obtains depot keys + * 2. Downloads and caches depot manifests + * 3. Allocates disk space for files + * 4. Downloads chunks concurrently (configured by [maxDownloads]) + * 5. Verifies checksums and moves files to final location + * + * ## Thread Safety + * Methods [add], [addAll], [addListener], and [removeListener] are thread-safe. + * Multiple concurrent calls are supported. + * + * @param steamClient Connected Steam client instance with valid login session + * @param licenses User's license list from [LicenseListCallback]. Required to determine which depots the account has access to. + * @param debug Enables detailed logging of all operations via [LogManager] + * @param useLanCache Attempts to detect and use local Steam cache servers (e.g., LANCache) for faster downloads on local networks + * @param maxDownloads Number of concurrent chunk downloads. Automatically increased to 25 when a LAN cache is detected. Default: 8 + * @param androidEmulation Forces "Windows" as the default OS filter. Used when running Android games in PC emulators that expect Windows builds. * * @author Oxters * @author Lossy * @since Oct 29, 2024 */ @Suppress("unused") -class ContentDownloader @JvmOverloads constructor( +class DepotDownloader @JvmOverloads constructor( private val steamClient: SteamClient, - private val licenses: List, // To be provided from [LicenseListCallback] - private val debug: Boolean = false, // Enable debugging features, such as logging - private val useLanCache: Boolean = false, // Try and detect a lan cache server. - private var maxDownloads: Int = 8, // Max concurrent downloads + private val licenses: List, + private val debug: Boolean = false, + private val useLanCache: Boolean = false, + private var maxDownloads: Int = 8, + private val androidEmulation: Boolean = false, ) : Closeable { companion object { @@ -109,17 +125,13 @@ class ContentDownloader @JvmOverloads constructor( val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" } - // What is a PriorityQueue? - private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - private val items = CopyOnWriteArrayList(ArrayList()) + private val activeDownloads = AtomicInteger(0) private val listeners = CopyOnWriteArrayList() private var logger: Logger? = null - private val isStarted: AtomicBoolean = AtomicBoolean(false) - private val processingChannel = Channel(Channel.UNLIMITED) - private val remainingItems = AtomicInteger(0) + private var processingChannel = Channel(Channel.UNLIMITED) private val lastFileProgressUpdate = ConcurrentHashMap() private val progressUpdateInterval = 500L // ms @@ -127,7 +139,7 @@ class ContentDownloader @JvmOverloads constructor( private var cdnClientPool: CDNClientPool? = null - private var config: Config = Config() + private var config: Config = Config(androidEmulation = androidEmulation) // region [REGION] Private data classes. @@ -161,15 +173,21 @@ class ContentDownloader @JvmOverloads constructor( init { if (debug) { - logger = LogManager.getLogger(ContentDownloader::class.java) + logger = LogManager.getLogger(DepotDownloader::class.java) } logger?.debug("DepotDownloader launched with ${licenses.size} for account") steam3 = Steam3Session(steamClient, debug) + + // Launch the processing loop + scope.launch { + processItems() + } } // region [REGION] Downloading Operations + @Throws(IllegalStateException::class) suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { val details = requireNotNull( @@ -321,7 +339,7 @@ class ContentDownloader @JvmOverloads constructor( steam3!!.requestAppInfo(appId, true) } else { val contentName = getAppName(appId) - throw ContentDownloaderException("App $appId ($contentName) is not available from this account.") + throw DepotDownloaderException("App $appId ($contentName) is not available from this account.") } } @@ -405,12 +423,12 @@ class ContentDownloader @JvmOverloads constructor( } if (depotManifestIds.isEmpty() && !hasSpecificDepots) { - throw ContentDownloaderException("Couldn't find any depots to download for app $appId") + throw DepotDownloaderException("Couldn't find any depots to download for app $appId") } if (depotIdsFound.size < depotIdsExpected.size) { val remainingDepotIds = depotIdsExpected.subtract(depotIdsFound.toSet()) - throw ContentDownloaderException("Depot ${remainingDepotIds.joinToString(", ")} not listed for app $appId") + throw DepotDownloaderException("Depot ${remainingDepotIds.joinToString(", ")} not listed for app $appId") } } @@ -704,7 +722,7 @@ class ContentDownloader @JvmOverloads constructor( EAppInfoSection.Extended -> "extended" EAppInfoSection.Config -> "config" EAppInfoSection.Depots -> "depots" - else -> throw ContentDownloaderException("${section.name} not implemented") + else -> throw DepotDownloaderException("${section.name} not implemented") } val sectionKV = appInfo.children.firstOrNull { c -> c.name == sectionKey } @@ -1047,7 +1065,7 @@ class ContentDownloader @JvmOverloads constructor( ) ) { notifyListeners { listener -> - listener.onDepotProgress(depot.depotId, snapshot) + listener.onDepotProgress(snapshot) } } } @@ -1072,7 +1090,7 @@ class ContentDownloader @JvmOverloads constructor( ) } notifyListeners { listener -> - listener.onDepotProgress(depot.depotId, snapshot) + listener.onDepotProgress(snapshot) } } } @@ -1173,7 +1191,7 @@ class ContentDownloader @JvmOverloads constructor( handle.resize(file.totalSize) } } catch (e: IOException) { - throw ContentDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") + throw DepotDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") } neededChunks = ArrayList(file.chunks) @@ -1242,7 +1260,7 @@ class ContentDownloader @JvmOverloads constructor( try { newHandle.resize(file.totalSize) } catch (ex: IOException) { - throw ContentDownloaderException( + throw DepotDownloaderException( "Failed to resize file to expected size $fileFinalPath: ${ex.message}" ) } @@ -1269,7 +1287,7 @@ class ContentDownloader @JvmOverloads constructor( try { handle.resize(file.totalSize) } catch (ex: IOException) { - throw ContentDownloaderException( + throw DepotDownloaderException( "Failed to allocate file $fileFinalPath: ${ex.message}" ) } @@ -1483,9 +1501,8 @@ class ContentDownloader @JvmOverloads constructor( notifyListeners { listener -> listener.onFileProgress( - depotId = depot.depotId, - fileName = file.fileName, progress = FileProgress( + depotId = depot.depotId, fileName = file.fileName, bytesDownloaded = approximateBytesDownloaded, totalBytes = file.totalSize, @@ -1542,356 +1559,166 @@ class ContentDownloader @JvmOverloads constructor( // endregion - // region [REGION] Array Operations - - fun getItems(): List = items.toList() - - fun size(): Int = items.size - - fun isEmpty(): Boolean = items.isEmpty() - - fun get(index: Int): DownloadItem? = items.getOrNull(index) - - fun contains(item: DownloadItem): Boolean = items.contains(item) - - fun indexOf(item: DownloadItem): Int = items.indexOf(item) - - fun addAll(items: List) { - items.forEach(::add) - } + // region [REGION] Queue Operations + /** + * Add a singular item of either [AppItem], [PubFileItem], or [UgcItem] + */ fun add(item: DownloadItem) { - val index = items.size - items.add(item) - - if (isStarted.get()) { - remainingItems.incrementAndGet() - scope.launch { processingChannel.send(item) } - } - - notifyListeners { it.onItemAdded(item, index) } - } - - fun addFirst(item: DownloadItem) { - if (isStarted.get()) { - logger?.debug("Cannot add item when started.") - return - } - - try { - items.add(0, item) - notifyListeners { it.onItemAdded(item, 0) } - } catch (e: Exception) { - logger?.error(e) - } - } - - fun addAt(index: Int, item: DownloadItem): Boolean { - if (isStarted.get()) { - logger?.debug("Cannot addAt item when started.") - return false - } - - return try { - items.add(index, item) - notifyListeners { it.onItemAdded(item, index) } - true - } catch (e: IndexOutOfBoundsException) { - false - } - } - - fun removeFirst(): DownloadItem? { - if (isStarted.get()) { - logger?.debug("Cannot removeFirst item when started.") - return null - } - - return try { - if (items.isNotEmpty()) { - val item = items.removeAt(0) - notifyListeners { it.onItemRemoved(item, 0) } - item - } else { - null - } - } catch (e: IndexOutOfBoundsException) { - logger?.error(e) - null + runBlocking { + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item.appId) } } } - fun removeLast(): DownloadItem? { - if (isStarted.get()) { - logger?.debug("Cannot removeLast item when started.") - return null - } - - return try { - if (items.isNotEmpty()) { - val lastIndex = items.size - 1 - val item = items.removeAt(lastIndex) - notifyListeners { it.onItemRemoved(item, lastIndex) } - item - } else { - null - } - } catch (e: IndexOutOfBoundsException) { - logger?.error(e) - null - } - } - - fun remove(item: DownloadItem): Boolean { - if (isStarted.get()) { - logger?.debug("Cannot remove item when started.") - return false - } - - val index = items.indexOf(item) - return try { - if (index >= 0) { - items.removeAt(index) - notifyListeners { it.onItemRemoved(item, index) } - true - } else { - false + /** + * Add a list items of either [AppItem], [PubFileItem], or [UgcItem] + */ + fun addAll(items: List) { + runBlocking { + items.forEach { item -> + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item.appId) } } - } catch (e: IndexOutOfBoundsException) { - logger?.error(e) - false - } - } - - fun removeAt(index: Int): DownloadItem? { - if (isStarted.get()) { - logger?.debug("Cannot removeAt item when started.") - return null - } - - return try { - val item = items.removeAt(index) - notifyListeners { it.onItemRemoved(item, index) } - item - } catch (e: IndexOutOfBoundsException) { - null - } - } - - fun moveItem(fromIndex: Int, toIndex: Int): Boolean { - if (isStarted.get()) { - logger?.debug("Cannot moveItem item when started.") - return false - } - - return try { - val item = items.removeAt(fromIndex) - items.add(toIndex, item) - true - } catch (e: IndexOutOfBoundsException) { - false } } - fun clear() { - if (isStarted.get()) { - logger?.debug("Cannot clear item when started.") - return - } - - val oldItems = items.toList() - items.clear() - - notifyListeners { it.onQueueCleared(oldItems) } - } - - // endregion - /** - * Some android emulators prefer using "Windows", so this will set downloading to prefer the Windows version. + * Get the current queue size of pending items to be downloaded. */ - fun setAndroidEmulation(value: Boolean) { - if (isStarted.get()) { - logger?.error("Can't set android emulation value once started.") - return - } + fun queueSize(): Int = activeDownloads.get() - config = config.copy(androidEmulation = value) - - notifyListeners { it.onAndroidEmulation(config.androidEmulation) } - } + /** + * Get a boolean value if there are items in queue to be downloaded. + */ + fun isProcessing(): Boolean = activeDownloads.get() > 0 - @Throws(IllegalStateException::class) - fun start(): CompletableFuture = scope.future { - if (isStarted.getAndSet(true)) { - logger?.debug("Downloading already started.") - return@future false - } + // endregion - val initialItems = items.toList() - if (initialItems.isEmpty()) { - logger?.debug("No items to download") - return@future false + private suspend fun processItems() = coroutineScope { + if (useLanCache) { + ClientLancache.detectLancacheServer() } - // Send initial items - remainingItems.set(initialItems.size) - initialItems.forEach { processingChannel.send(it) } - if (ClientLancache.useLanCacheServer) { logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") - - // Increasing the number of concurrent downloads when the cache is detected since the downloads will likely - // be served much faster than over the internet. Steam internally has this behavior as well. if (maxDownloads == 8) { maxDownloads = 25 } } - repeat(remainingItems.get()) { - // Process exactly this many - ensureActive() - - // Obtain the next item in queue. - val item = processingChannel.receive() - + for (item in processingChannel) { try { - runBlocking { - if (useLanCache) { - ClientLancache.detectLancacheServer() - } - - // Set some configuration values, first. - config = config.copy( - downloadManifestOnly = item.downloadManifestOnly, - installPath = item.installDirectory?.toPath(), - installToGameNameDirectory = item.installToGameNameDirectory, - ) - - // Sequential looping. - when (item) { - is PubFileItem -> { - if (item.pubfile == INVALID_MANIFEST_ID) { - logger?.debug("Invalid Pub File ID for ${item.appId}") - return@runBlocking - } + ensureActive() - logger?.debug("Downloading PUB File for ${item.appId}") + // Set configuration values + config = config.copy( + downloadManifestOnly = item.downloadManifestOnly, + installPath = item.installDirectory?.toPath(), + installToGameNameDirectory = item.installToGameNameDirectory, + ) - notifyListeners { it.onDownloadStarted(item) } - downloadPubFile(item.appId, item.pubfile) + when (item) { + is PubFileItem -> { + if (item.pubfile == INVALID_MANIFEST_ID) { + logger?.debug("Invalid Pub File ID for ${item.appId}") + continue } + logger?.debug("Downloading PUB File for ${item.appId}") + notifyListeners { it.onDownloadStarted(item.appId) } + downloadPubFile(item.appId, item.pubfile) + } - is UgcItem -> { - if (item.ugcId == INVALID_MANIFEST_ID) { - logger?.debug("Invalid UGC ID for ${item.appId}") - return@runBlocking - } - - logger?.debug("Downloading UGC File for ${item.appId}") - - notifyListeners { it.onDownloadStarted(item) } - downloadUGC(item.appId, item.ugcId) + is UgcItem -> { + if (item.ugcId == INVALID_MANIFEST_ID) { + logger?.debug("Invalid UGC ID for ${item.appId}") + continue } + logger?.debug("Downloading UGC File for ${item.appId}") + notifyListeners { it.onDownloadStarted(item.appId) } + downloadUGC(item.appId, item.ugcId) + } - is AppItem -> { - val branch = item.branch ?: DEFAULT_BRANCH - config = config.copy(betaPassword = item.branchPassword) - - if (!config.betaPassword.isNullOrBlank() && branch.isBlank()) { - logger?.error("Error: Cannot specify 'branchpassword' when 'branch' is not specified.") - return@runBlocking - } - - config = config.copy(downloadAllPlatforms = item.downloadAllPlatforms) - - val os = item.os - - if (config.downloadAllPlatforms && !os.isNullOrBlank()) { - logger?.error("Error: Cannot specify `os` when `all-platforms` is specified.") - return@runBlocking - } - - config = config.copy(downloadAllArchs = item.downloadAllArchs) + is AppItem -> { + val branch = item.branch ?: DEFAULT_BRANCH + config = config.copy(betaPassword = item.branchPassword) - val arch = item.osArch + if (!config.betaPassword.isNullOrBlank() && branch.isBlank()) { + logger?.error("Error: Cannot specify 'branchpassword' when 'branch' is not specified.") + continue + } - if (config.downloadAllArchs && !arch.isNullOrBlank()) { - logger?.error("Error: Cannot specify `osarch` when `all-archs` is specified.") - return@runBlocking - } + config = config.copy(downloadAllPlatforms = item.downloadAllPlatforms) + val os = item.os - config = config.copy(downloadAllLanguages = item.downloadAllLanguages) + if (config.downloadAllPlatforms && !os.isNullOrBlank()) { + logger?.error("Error: Cannot specify `os` when `all-platforms` is specified.") + continue + } - val language = item.language + config = config.copy(downloadAllArchs = item.downloadAllArchs) + val arch = item.osArch - if (config.downloadAllLanguages && !language.isNullOrBlank()) { - logger?.error("Error: Cannot specify `language` when `all-languages` is specified.") - return@runBlocking - } + if (config.downloadAllArchs && !arch.isNullOrBlank()) { + logger?.error("Error: Cannot specify `osarch` when `all-archs` is specified.") + continue + } - val lv = item.lowViolence + config = config.copy(downloadAllLanguages = item.downloadAllLanguages) + val language = item.language - val depotManifestIds = mutableListOf>() - val isUGC = false + if (config.downloadAllLanguages && !language.isNullOrBlank()) { + logger?.error("Error: Cannot specify `language` when `all-languages` is specified.") + continue + } - val depotIdList = item.depot - val manifestIdList = item.manifest + val depotManifestIds = mutableListOf>() + val depotIdList = item.depot + val manifestIdList = item.manifest - if (manifestIdList.isNotEmpty()) { - if (depotIdList.size != manifestIdList.size) { - logger?.error("Error: `manifest` requires one id for every `depot` specified") - return@runBlocking - } - val zippedDepotManifest = depotIdList.zip(manifestIdList) { depotId, manifestId -> - Pair(depotId, manifestId) - } - depotManifestIds.addAll(zippedDepotManifest) - } else { - depotManifestIds.addAll( - depotIdList.map { depotId -> - Pair(depotId, INVALID_MANIFEST_ID) - } - ) + if (manifestIdList.isNotEmpty()) { + if (depotIdList.size != manifestIdList.size) { + logger?.error("Error: `manifest` requires one id for every `depot` specified") + continue } - - logger?.debug("Downloading App for ${item.appId}") - - notifyListeners { it.onDownloadStarted(item) } - downloadApp( - appId = item.appId, - depotManifestIds = depotManifestIds, - branch = branch, - os = os, - arch = arch, - language = language, - lv = lv, - isUgc = isUGC, - ) + depotManifestIds.addAll(depotIdList.zip(manifestIdList)) + } else { + depotManifestIds.addAll(depotIdList.map { it to INVALID_MANIFEST_ID }) } - } - notifyListeners { it.onDownloadCompleted(item) } + logger?.debug("Downloading App for ${item.appId}") + notifyListeners { it.onDownloadStarted(item.appId) } + downloadApp( + appId = item.appId, + depotManifestIds = depotManifestIds, + branch = branch, + os = os, + arch = arch, + language = language, + lv = item.lowViolence, + isUgc = false, + ) + } } - } catch (e: IOException) { - logger?.error("Error downloading item ${item.appId}: ${e.message}", e) - - notifyListeners { it.onDownloadFailed(item, e) } - throw e + notifyListeners { it.onDownloadCompleted(item.appId) } + } catch (e: Exception) { + logger?.error("Error downloading item ${item.appId}: ${e.message}", e) + notifyListeners { it.onDownloadFailed(item.appId, e) } + } finally { + activeDownloads.decrementAndGet() } } - - return@future true } override fun close() { - isStarted.set(false) + processingChannel.close() - HttpClient.close() + scope.cancel("DepotDownloader Closing") - items.clear() - processingChannel.close() + HttpClient.close() lastFileProgressUpdate.clear() listeners.clear() diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt new file mode 100644 index 00000000..4c3b4bbc --- /dev/null +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloaderException.kt @@ -0,0 +1,11 @@ +package `in`.dragonbra.javasteam.depotdownloader + +/** + * Exception thrown when content download operations fail. + * Used to indicate errors during depot downloads, manifest retrieval, + * or other content downloader operations. + * + * @author Lossy + * @since Oct 1, 2025 + */ +class DepotDownloaderException(value: String) : Exception(value) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 340f85a9..75e4f612 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -8,6 +8,11 @@ import io.ktor.client.plugins.UserAgent import kotlinx.coroutines.isActive /** + * Singleton HTTP client for content downloader operations. + * Provides a shared, configured Ktor HTTP client optimized for Steam CDN downloads. + * The client is lazily initialized on first use and reused across all download operations. + * Connection pooling and timeouts are configured based on the maximum concurrent downloads. + * * @author Lossy * @since Oct 1, 2025 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt index c3ce3c22..9b4ab7f2 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -1,35 +1,79 @@ package `in`.dragonbra.javasteam.depotdownloader import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress -import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress /** - * Listener interface for download events. + * Listener interface for receiving download progress and status events. + * + * All methods have default empty implementations, allowing listeners to + * implement only the callbacks they need. * * @author Lossy * @since Oct 1, 2025 */ interface IDownloadListener { - // Queue management - fun onItemAdded(item: DownloadItem, index: Int) {} - fun onItemRemoved(item: DownloadItem, index: Int) {} - fun onQueueCleared(previousItems: List) {} + /** + * Called when an item is added to the download queue. + * + * @param appId The application ID that was queued + */ + fun onItemAdded(appId: Int) {} + + /** + * Called when a download begins processing. + * + * @param appId The application ID being downloaded + */ + fun onDownloadStarted(appId: Int) {} + + /** + * Called when a download completes successfully. + * + * @param appId The application ID that finished downloading + */ + fun onDownloadCompleted(appId: Int) {} - // Download lifecycle - fun onDownloadStarted(item: DownloadItem) {} - fun onDownloadCompleted(item: DownloadItem) {} - fun onDownloadFailed(item: DownloadItem, error: Throwable) {} + /** + * Called when a download fails with an error. + * + * @param appId The application ID that failed + * @param error The exception that caused the failure + */ + fun onDownloadFailed(appId: Int, error: Throwable) {} - // Progress tracking + /** + * Called periodically with overall download progress across all items. + * Reports progress for the entire download queue, including completed + * and remaining items. + * + * @param progress Overall download statistics + */ fun onOverallProgress(progress: OverallProgress) {} - fun onDepotProgress(depotId: Int, progress: DepotProgress) {} - fun onFileProgress(depotId: Int, fileName: String, progress: FileProgress) {} - // Status updates - fun onStatusUpdate(message: String) {} + /** + * Called periodically with progress for a specific depot. + * Reports file allocation and download progress for an individual depot. + * + * @param progress Depot-specific download statistics + */ + fun onDepotProgress(progress: DepotProgress) {} + + /** + * Called periodically with progress for a specific file. + * Reports chunk-level download progress for individual files. + * + * @param progress File-specific download statistics + */ + fun onFileProgress(progress: FileProgress) {} - // Configuration - fun onAndroidEmulation(value: Boolean) {} + /** + * Called with informational status messages during download operations. + * Used for logging or displaying current operations like manifest + * downloads, file validation, and allocation. + * + * @param message Human-readable status message + */ + fun onStatusUpdate(message: String) {} } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 3f7e7099..9a3ebb94 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -28,6 +28,14 @@ import java.io.Closeable import java.util.concurrent.ConcurrentHashMap /** + * Manages Steam protocol session state and API interactions for content downloading. + * + * All Steam API handlers are initialized during construction and must remain valid + * for the lifetime of this session. + * + * @param steamClient The connected Steam client instance + * @param debug If true, enables debug logging for all Steam API operations + * * @author Lossy * @since Oct 1, 2025 */ @@ -304,7 +312,7 @@ class Steam3Session( return privateBeta.depotSection } - @Throws(ContentDownloaderException::class) + @Throws(DepotDownloaderException::class) suspend fun getPublishedFileDetails( appId: Int, pubFile: PublishedFileID, @@ -323,7 +331,7 @@ class Steam3Session( return details.body.publishedfiledetailsBuilderList.firstOrNull()?.build() } - throw ContentDownloaderException("EResult ${details.result.code()} (${details.result}) while retrieving file details for pubfile $pubFile.") + throw DepotDownloaderException("EResult ${details.result.code()} (${details.result}) while retrieving file details for pubfile $pubFile.") } suspend fun getUGCDetails(ugcHandle: UGCHandle): UGCDetailsCallback? { @@ -337,6 +345,6 @@ class Steam3Session( return null } - throw ContentDownloaderException("EResult ${callback.result.code()} (${callback.result}) while retrieving UGC details for ${ugcHandle.value}.") + throw DepotDownloaderException("EResult ${callback.result.code()} (${callback.result}) while retrieving UGC details for ${ugcHandle.value}.") } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt index 05fbb247..dafee0bd 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/ChunkMatch.kt @@ -3,6 +3,13 @@ package `in`.dragonbra.javasteam.depotdownloader.data import `in`.dragonbra.javasteam.types.ChunkData /** + * Pairs matching chunks between old and new depot manifests for differential updates. + * Used during file validation to identify chunks that can be reused from existing + * files, avoiding unnecessary re-downloads when only portions of a file have changed. + * + * @property oldChunk Chunk from the previously installed manifest + * @property newChunk Corresponding chunk from the new manifest being downloaded + * * @author Oxters * @since Oct 29, 2024 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt index 256575ec..5b853ce4 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotDownloadInfo.kt @@ -3,6 +3,18 @@ package `in`.dragonbra.javasteam.depotdownloader.data import okio.Path /** + * Contains all information required to download a specific depot manifest and its content. + * This class aggregates the depot identification, authentication, and installation details + * needed to perform a complete depot download operation. It is created during the depot + * resolution phase and passed through the download pipeline. + * + * @property depotId The Steam depot identifier + * @property appId The owning application ID (may differ from the app being downloaded if the depot uses `depotfromapp` proxying) + * @property manifestId The specific manifest version to download + * @property branch The branch name this manifest belongs to (e.g., "public", "beta") + * @property installDir The target directory for downloaded files + * @property depotKey The AES decryption key for depot chunks. Cleared on disposal for security. + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt index bef4590b..15d4dda9 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DepotFilesData.kt @@ -5,6 +5,18 @@ import `in`.dragonbra.javasteam.types.FileData import okio.Path /** + * Aggregates all data needed to process and download files for a single depot. + * Created during manifest processing and passed to the download phase. Contains both + * the current manifest and optional previous manifest to enable differential updates. + * + * @property depotDownloadInfo Core depot identification and authentication details + * @property depotCounter Progress tracking counters for this depot's download + * @property stagingDir Temporary directory for in-progress file writes + * @property manifest The current depot manifest being downloaded + * @property previousManifest The previously installed manifest, if any. Used to identify reusable chunks and deleted files. + * @property filteredFiles Files to download after applying platform, language, and user filters. Modified during processing to remove duplicates across depots. + * @property allFileNames Complete set of filenames in this depot, including directories. Used for cross-depot deduplication and cleanup of deleted files. + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt index 8d509e27..380463c2 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadCounters.kt @@ -3,6 +3,14 @@ package `in`.dragonbra.javasteam.depotdownloader.data // https://kotlinlang.org/docs/coding-conventions.html#source-file-organization /** + * Tracks cumulative download statistics across all depots in a download session. + * Used for overall progress reporting and final download summary. All fields are + * accessed under synchronization to ensure thread-safe updates from concurrent downloads. + * + * @property completeDownloadSize Total bytes expected to download across all depots. Adjusted during validation when existing chunks are reused. + * @property totalBytesCompressed Total compressed bytes transferred from CDN servers + * @property totalBytesUncompressed Total uncompressed bytes written to disk + * * @author Oxters * @author Lossy * @since Oct 29, 2024 @@ -14,6 +22,15 @@ data class GlobalDownloadCounter( ) /** + * Tracks download statistics for a single depot. + * Used for depot-level progress reporting. All fields are accessed under synchronization + * to ensure thread-safe updates from concurrent chunk downloads. + * + * @property completeDownloadSize Total bytes expected to download for this depot. Calculated during file allocation phase. + * @property sizeDownloaded Bytes successfully downloaded and written so far + * @property depotBytesCompressed Compressed bytes transferred from CDN for this depot + * @property depotBytesUncompressed Uncompressed bytes written to disk for this depot + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt index a204c4e1..e94ecdd6 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -1,10 +1,14 @@ package `in`.dragonbra.javasteam.depotdownloader.data -import `in`.dragonbra.javasteam.depotdownloader.ContentDownloader - // https://kotlinlang.org/docs/coding-conventions.html#source-file-organization /** + * Base class for downloadable Steam content items. + * @property appId The Steam application ID + * @property installDirectory Optional custom installation directory path + * @property installToGameNameDirectory If true, installs to a directory named after the game + * @property downloadManifestOnly If true, only downloads the manifest file without actual content + * * @author Lossy * @since Oct 1, 2025 */ @@ -16,30 +20,52 @@ abstract class DownloadItem( ) /** + * Represents a Steam Workshop (UGC - User Generated Content) item for download. + * + * @property ugcId The unique UGC item identifier + * * @author Lossy * @since Oct 1, 2025 */ class UgcItem @JvmOverloads constructor( appId: Int, - val ugcId: Long = ContentDownloader.INVALID_MANIFEST_ID, + val ugcId: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, downloadManifestOnly: Boolean = false, ) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) /** + * Represents a Steam published file for download. + * + * @property pubfile The published file identifier + * * @author Lossy * @since Oct 1, 2025 */ class PubFileItem @JvmOverloads constructor( appId: Int, - val pubfile: Long = ContentDownloader.INVALID_MANIFEST_ID, + val pubfile: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, downloadManifestOnly: Boolean = false, ) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) /** + * Represents a Steam application/game for download from a depot. + * + * @property branch The branch name to download from (e.g., "public", "beta") + * @property branchPassword Password for password-protected branches + * @property downloadAllPlatforms If true, downloads depots for all platforms + * @property os Operating system filter (e.g., "windows", "macos", "linux") + * @property downloadAllArchs If true, downloads depots for all architectures + * @property osArch Architecture filter (e.g., "32", "64") + * @property downloadAllLanguages If true, downloads depots for all languages + * @property language Language filter (e.g., "english", "french") + * @property lowViolence If true, downloads low-violence versions where available + * @property depot List of specific depot IDs to download + * @property manifest List of specific manifest IDs corresponding to depot IDs + * * @author Lossy * @since Oct 1, 2025 */ diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt index f8e8035d..2e058560 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt @@ -1,7 +1,16 @@ package `in`.dragonbra.javasteam.depotdownloader.data /** - * Overall download progress across all items. + * Reports overall download progress across all queued items. + * Provides high-level statistics for the entire download session, tracking + * which item is currently processing and cumulative byte transfer. + * + * @property currentItem Number of items completed (1-based) + * @property totalItems Total number of items in the download session + * @property totalBytesDownloaded Cumulative uncompressed bytes downloaded across all depots + * @property totalBytesExpected Total uncompressed bytes expected for all items + * @property status Current download phase + * @property percentComplete Calculated completion percentage (0.0 to 100.0) * * @author Lossy * @since Oct 1, 2025 @@ -22,7 +31,18 @@ data class OverallProgress( } /** - * Progress for a specific depot + * Reports download progress for a specific depot within an item. + * Tracks both file-level progress (allocation/validation) and byte-level + * download progress. During the [DownloadStatus.PREPARING] phase, tracks + * file allocation; during [DownloadStatus.DOWNLOADING], tracks actual transfers. + * + * @property depotId The Steam depot identifier + * @property filesCompleted Number of files fully allocated or downloaded + * @property totalFiles Total files to process in this depot (excludes directories) + * @property bytesDownloaded Uncompressed bytes successfully downloaded + * @property totalBytes Total uncompressed bytes expected for this depot + * @property status Current depot processing phase + * @property percentComplete Calculated completion percentage (0.0 to 100.0) * * @author Lossy * @since Oct 1, 2025 @@ -44,12 +64,24 @@ data class DepotProgress( } /** - * Progress for a specific file + * Reports download progress for an individual file. + * Provides chunk-level granularity for tracking file downloads. Updates are + * throttled to every 500ms to avoid excessive callback overhead. + * + * @property depotId The Steam depot containing this file + * @property fileName Relative path of the file within the depot + * @property bytesDownloaded Approximate uncompressed bytes downloaded (based on chunk completion) + * @property totalBytes Total uncompressed file size + * @property chunksCompleted Number of chunks successfully downloaded and written + * @property totalChunks Total chunks comprising this file + * @property status Current file download status + * @property percentComplete Calculated completion percentage (0.0 to 100.0) * * @author Lossy * @since Oct 1, 2025 */ data class FileProgress( + val depotId: Int, val fileName: String, val bytesDownloaded: Long, val totalBytes: Long, @@ -65,6 +97,13 @@ data class FileProgress( } } +/** + * Represents the current phase of a download operation. + * + * @property PREPARING File allocation and validation phase. Files are being pre-allocated on disk and existing content is being verified. + * @property DOWNLOADING Active chunk download phase. Content is being transferred from CDN. + * @property COMPLETED Download finished successfully. All files written and verified. + */ enum class DownloadStatus { PREPARING, DOWNLOADING, diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt index 2b6a539c..1bc68fac 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/FileStreamData.kt @@ -5,6 +5,14 @@ import okio.FileHandle import java.util.concurrent.atomic.AtomicInteger /** + * Internal state for managing concurrent chunk writes to a single file. + * Coordinates writes from multiple download workers to ensure thread-safe file access + * and tracks when all chunks have been written. + * + * @property fileHandle Shared file handle for all chunk writes. Lazily opened on first write. + * @property fileLock Mutex protecting concurrent access to [fileHandle] + * @property chunksToDownload Atomic counter of remaining chunks. File is closed when this reaches zero. + * * @author Oxters * @author Lossy * @since Oct 29, 2024 diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index b73f408a..f7abc981 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -1,6 +1,6 @@ package in.dragonbra.javasteamsamples._023_downloadapp; -import in.dragonbra.javasteam.depotdownloader.ContentDownloader; +import in.dragonbra.javasteam.depotdownloader.DepotDownloader; import in.dragonbra.javasteam.depotdownloader.IDownloadListener; import in.dragonbra.javasteam.depotdownloader.data.*; import in.dragonbra.javasteam.enums.EResult; @@ -30,8 +30,8 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.Scanner; import java.util.concurrent.CancellationException; -import java.util.concurrent.ExecutionException; /** @@ -123,6 +123,7 @@ public void run() { for (var subscription : subscriptions) { try { + System.out.println("Closing: " + subscription.getClass().getName()); subscription.close(); } catch (IOException e) { System.out.println("Couldn't close a callback."); @@ -238,7 +239,6 @@ private void onLicenseList(LicenseListCallback callback) { System.out.println("Got " + licenseList.size() + " licenses from account!"); } - @SuppressWarnings("ExtractMethodRecommender") private void onFreeLicense(FreeLicenseCallback callback) { if (callback.getResult() != EResult.OK) { System.out.println("Failed to get a free license for Rocky Mayhem"); @@ -248,69 +248,72 @@ private void onFreeLicense(FreeLicenseCallback callback) { // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. // You will need to subscribe to LicenseListCallback to obtain your app licenses. - try (var depotDownloader = new ContentDownloader(steamClient, licenseList, false)) { + try (var depotDownloader = new DepotDownloader(steamClient, licenseList, false)) { + // Add this class as a listener of IDownloadListener depotDownloader.addListener(this); // An app id is required at minimum for all item types. var pubItem = new PubFileItem( /* appId */ 0, /* pubfile */ 0, - /* installToGameNameDirectory */ false, - /* installDirectory */ null, - /* downloadManifestOnly */ false + /* (Optional) installToGameNameDirectory */ false, + /* (Optional) installDirectory */ null, + /* (Optional) downloadManifestOnly */ false ); // TODO find actual pub item var ugcItem = new UgcItem( /* appId */0, /* ugcId */ 0, - /* installToGameNameDirectory */ false, - /* installDirectory */ null, - /* downloadManifestOnly */ false + /* (Optional) installToGameNameDirectory */ false, + /* (Optional) installDirectory */ null, + /* (Optional) downloadManifestOnly */ false ); // TODO find actual ugc item var appItem = new AppItem( /* appId */ 204360, - /* installToGameNameDirectory */ true, - /* installDirectory */ DEFAULT_INSTALL_DIRECTORY, - /* branch */ "public", - /* branchPassword */ "", - /* downloadAllPlatforms */ false, - /* os */ "windows", - /* downloadAllArchs */ false, - /* osArch */ "64", - /* downloadAllLanguages */ false, - /* language */ "english", - /* lowViolence */ false, - /* depot */ List.of(), - /* manifest */ List.of(), - /* downloadManifestOnly */ false + /* (Optional) installToGameNameDirectory */ true, + /* (Optional) installDirectory */ DEFAULT_INSTALL_DIRECTORY, + /* (Optional) branch */ "public", + /* (Optional) branchPassword */ "", + /* (Optional) downloadAllPlatforms */ false, + /* (Optional) os */ "windows", + /* (Optional) downloadAllArchs */ false, + /* (Optional) osArch */ "64", + /* (Optional) downloadAllLanguages */ false, + /* (Optional) language */ "english", + /* (Optional) lowViolence */ false, + /* (Optional) depot */ List.of(), + /* (Optional) manifest */ List.of(), + /* (Optional) downloadManifestOnly */ false ); - var appItem2 = new AppItem(225840, true); - var appItem3 = new AppItem(3527290, true); - var appItem4 = new AppItem(ROCKY_MAYHEM_APP_ID, true); + var scanner = new Scanner(System.in); + System.out.print("Enter a game app id: "); + var appId = scanner.nextInt(); - var downloadList = List.of(pubItem, ugcItem, appItem); + // After 'depotDownloader' is constructed, items added are downloaded in a First-In, First-Out queue on the fly. - // Add specified games to the queue. Add, Remove, Move, and general array manipulation methods are available. - // depotDownloader.addAll(downloadList); // TODO - depotDownloader.addAll(List.of(appItem)); + // Add a singular item to process. + depotDownloader.add(new AppItem(appId, true)); - // Start downloading your items. Array manipulation is now disabled. You can still add to the list. - var success = depotDownloader.start().get(); // Future + // You can add a List of items to be processed. + // depotDownloader.add(List.of()); - if (success) { - System.out.println("Download completed successfully"); + // Stay here while content downloads. Note this sample is synchronous so we'll loop here. + while (depotDownloader.isProcessing()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + break; + } } + // Remove this class as a listener of IDownloadListener depotDownloader.removeListener(this); - } catch (IllegalStateException | InterruptedException | ExecutionException e) { - System.out.println("Something happened"); - System.err.println(e.getMessage()); } finally { - steamUser.logOff(); System.out.println("Done Downloading"); + steamUser.logOff(); } } @@ -323,38 +326,26 @@ private void onLoggedOff(LoggedOffCallback callback) { // Depot Downloader Callbacks. @Override - public void onItemAdded(@NotNull DownloadItem item, int index) { - System.out.println("Depot Downloader: Item Added: " + item.getAppId() + ", index: " + index); - System.out.println(" ---- "); - } - - @Override - public void onItemRemoved(@NotNull DownloadItem item, int index) { - System.out.println("Depot Downloader: Item Removed: " + item.getAppId() + ", index: " + index); + public void onItemAdded(int appId) { + System.out.println("Depot Downloader: Item Added: " + appId); System.out.println(" ---- "); } @Override - public void onQueueCleared(@NotNull List previousItems) { - System.out.println("Depot Downloader: Queue size of " + previousItems.size() + " cleared"); + public void onDownloadStarted(int appId) { + System.out.println("Depot Downloader: Download started for item: " + appId); System.out.println(" ---- "); } @Override - public void onDownloadStarted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download started for item: " + item.getAppId()); + public void onDownloadCompleted(int appId) { + System.out.println("Depot Downloader: Download completed for item: " + appId); System.out.println(" ---- "); } @Override - public void onDownloadCompleted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download completed for item: " + item.getAppId()); - System.out.println(" ---- "); - } - - @Override - public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { - System.out.println("Depot Downloader: Download failed for item: " + item.getAppId()); + public void onDownloadFailed(int appId, @NotNull Throwable error) { + System.out.println("Depot Downloader: Download failed for item: " + appId); System.err.println(error.getMessage()); System.out.println(" ---- "); } @@ -372,9 +363,8 @@ public void onOverallProgress(@NotNull OverallProgress progress) { } @Override - public void onDepotProgress(int depotId, @NotNull DepotProgress progress) { + public void onDepotProgress(@NotNull DepotProgress progress) { System.out.println("Depot Downloader: Depot Progress"); - System.out.println("depotId: " + depotId); System.out.println("depotId: " + progress.getDepotId()); System.out.println("filesCompleted: " + progress.getFilesCompleted()); System.out.println("totalFiles: " + progress.getTotalFiles()); @@ -386,10 +376,9 @@ public void onDepotProgress(int depotId, @NotNull DepotProgress progress) { } @Override - public void onFileProgress(int depotId, @NotNull String fileName, @NotNull FileProgress progress) { + public void onFileProgress(@NotNull FileProgress progress) { System.out.println("Depot Downloader: File Progress"); - System.out.println("depotId: " + depotId); - System.out.println("fileName: " + fileName); + System.out.println("depotId: " + progress.getDepotId()); System.out.println("fileName: " + progress.getFileName()); System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); System.out.println("totalBytes: " + progress.getTotalBytes()); @@ -405,10 +394,4 @@ public void onStatusUpdate(@NotNull String message) { System.out.println("Depot Downloader: Status Message: " + message); System.out.println(" ---- "); } - - @Override - public void onAndroidEmulation(boolean value) { - System.out.println("Depot Downloader: Android Emulation: " + value); - System.out.println(" ---- "); - } } From a9aa8d86e0362ef496ab71a5044ab9146e4d0a8b Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 5 Oct 2025 02:23:15 -0500 Subject: [PATCH 08/44] Simplify SampleDownloadApp --- .../_023_downloadapp/SampleDownloadApp.java | 48 +++++++------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index f7abc981..9d25aaf1 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -9,8 +9,6 @@ import in.dragonbra.javasteam.steam.authentication.AuthenticationException; import in.dragonbra.javasteam.steam.authentication.UserConsoleAuthenticator; import in.dragonbra.javasteam.steam.handlers.steamapps.License; -import in.dragonbra.javasteam.steam.handlers.steamapps.SteamApps; -import in.dragonbra.javasteam.steam.handlers.steamapps.callback.FreeLicenseCallback; import in.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback; import in.dragonbra.javasteam.steam.handlers.steamuser.LogOnDetails; import in.dragonbra.javasteam.steam.handlers.steamuser.SteamUser; @@ -42,27 +40,21 @@ *

* this sample introduces the usage of the content downloader API *

- * content downloader lets you download an app from a Steam depot given - * an app ID + * content downloader lets you download an app, pub file, or ugc item given some parameters. *

- * in this case, this sample will demonstrate how to download the free game - * called Rocky Mayhem + * in this case, this sample will ask which game app id you'd like to download. + * You can find the app id of a game by the url of the store page. + * For example "store.steampowered.com/app/1303350/Rocky_Mayhem/", where 1303350 is the app id. */ @SuppressWarnings("FieldCanBeLocal") public class SampleDownloadApp implements Runnable, IDownloadListener { - private final int ROCKY_MAYHEM_APP_ID = 1303350; - - private final String DEFAULT_INSTALL_DIRECTORY = "steamapps"; - private SteamClient steamClient; private CallbackManager manager; private SteamUser steamUser; - private SteamApps steamApps; - private boolean isRunning; private final String user; @@ -100,8 +92,6 @@ public void run() { steamUser = steamClient.getHandler(SteamUser.class); - steamApps = steamClient.getHandler(SteamApps.class); - subscriptions = new ArrayList<>(); subscriptions.add(manager.subscribe(ConnectedCallback.class, this::onConnected)); @@ -109,7 +99,6 @@ public void run() { subscriptions.add(manager.subscribe(LoggedOnCallback.class, this::onLoggedOn)); subscriptions.add(manager.subscribe(LoggedOffCallback.class, this::onLoggedOff)); subscriptions.add(manager.subscribe(LicenseListCallback.class, this::onLicenseList)); - subscriptions.add(manager.subscribe(FreeLicenseCallback.class, this::onFreeLicense)); isRunning = true; @@ -121,9 +110,9 @@ public void run() { manager.runWaitCallbacks(1000L); } + System.out.println("Closing " + subscriptions.size() + " subscriptions."); for (var subscription : subscriptions) { try { - System.out.println("Closing: " + subscription.getClass().getName()); subscription.close(); } catch (IOException e) { System.out.println("Couldn't close a callback."); @@ -223,8 +212,9 @@ private void onLoggedOn(LoggedOnCallback callback) { System.out.println("Successfully logged on!"); - // now that we are logged in, we can request a free license for Rocky Mayhem - steamApps.requestFreeLicense(ROCKY_MAYHEM_APP_ID); + // at this point, we'd be able to perform actions on Steam + + // The sample continues in onLicenseList } private void onLicenseList(LicenseListCallback callback) { @@ -237,15 +227,17 @@ private void onLicenseList(LicenseListCallback callback) { licenseList = callback.getLicenseList(); System.out.println("Got " + licenseList.size() + " licenses from account!"); + + downloadApp(); } - private void onFreeLicense(FreeLicenseCallback callback) { - if (callback.getResult() != EResult.OK) { - System.out.println("Failed to get a free license for Rocky Mayhem"); - steamClient.disconnect(); - return; - } + private void onLoggedOff(LoggedOffCallback callback) { + System.out.println("Logged off of Steam: " + callback.getResult()); + isRunning = false; + } + + private void downloadApp() { // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. // You will need to subscribe to LicenseListCallback to obtain your app licenses. try (var depotDownloader = new DepotDownloader(steamClient, licenseList, false)) { @@ -273,7 +265,7 @@ private void onFreeLicense(FreeLicenseCallback callback) { var appItem = new AppItem( /* appId */ 204360, /* (Optional) installToGameNameDirectory */ true, - /* (Optional) installDirectory */ DEFAULT_INSTALL_DIRECTORY, + /* (Optional) installDirectory */ "steamapps", /* (Optional) branch */ "public", /* (Optional) branchPassword */ "", /* (Optional) downloadAllPlatforms */ false, @@ -317,12 +309,6 @@ private void onFreeLicense(FreeLicenseCallback callback) { } } - private void onLoggedOff(LoggedOffCallback callback) { - System.out.println("Logged off of Steam: " + callback.getResult()); - - isRunning = false; - } - // Depot Downloader Callbacks. @Override From c2e9e249327f78283aa7834672e790417e15d3a1 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 5 Oct 2025 19:34:54 -0500 Subject: [PATCH 09/44] A litte more tidying --- .../depotdownloader/CDNClientPool.kt | 42 ++++--- .../depotdownloader/DepotDownloader.kt | 83 +++++++------ .../javasteam/depotdownloader/HttpClient.kt | 29 +++-- .../depotdownloader/IDownloadListener.kt | 17 +-- .../depotdownloader/data/DownloadItems.kt | 12 +- .../_023_downloadapp/SampleDownloadApp.java | 112 +++++++++++------- .../in/dragonbra/javasteam/types/FileData.kt | 29 ++++- 7 files changed, 210 insertions(+), 114 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index 58ac5366..bd24d98d 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -7,6 +7,8 @@ import `in`.dragonbra.javasteam.util.log.Logger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference /** * Manages a pool of CDN server connections for efficient content downloading. @@ -32,9 +34,9 @@ class CDNClientPool( private var logger: Logger? = null - private val servers: ArrayList = arrayListOf() + private val servers = AtomicReference>(emptyList()) - private var nextServer: Int = 0 + private var nextServer: AtomicInteger = AtomicInteger(0) private val mutex: Mutex = Mutex() @@ -53,7 +55,7 @@ class CDNClientPool( } override fun close() { - servers.clear() + servers.set(emptyList()) cdnClient = null proxyServer = null @@ -63,10 +65,6 @@ class CDNClientPool( @Throws(Exception::class) suspend fun updateServerList(maxNumServers: Int? = null) = mutex.withLock { - if (servers.isNotEmpty()) { - servers.clear() - } - val serversForSteamPipe = steamSession.steamContent!!.getServersForSteamPipe( cellId = steamSession.steamClient.cellID ?: 0, maxNumServers = maxNumServers, @@ -84,27 +82,32 @@ class CDNClientPool( // ContentServerPenalty removed for now. - servers.addAll(weightedCdnServers) + servers.set(weightedCdnServers) + + nextServer.set(0) // servers.joinToString(separator = "\n", prefix = "Servers:\n") { "- $it" } - logger?.debug("Found ${servers.size} Servers") + logger?.debug("Found ${weightedCdnServers.size} Servers") - if (servers.isEmpty()) { + if (weightedCdnServers.isEmpty()) { throw Exception("Failed to retrieve any download servers.") } } - suspend fun getConnection(): Server = mutex.withLock { - val server = servers[nextServer % servers.count()] + fun getConnection(): Server { + val servers = servers.get() + + val index = nextServer.getAndIncrement() + val server = servers[index % servers.size] logger?.debug("Getting connection $server") return server } - suspend fun returnConnection(server: Server?) = mutex.withLock { + fun returnConnection(server: Server?) { if (server == null) { - return@withLock + return } logger?.debug("Returning connection: $server") @@ -112,15 +115,18 @@ class CDNClientPool( // (SK) nothing to do, maybe remove from ContentServerPenalty? } - suspend fun returnBrokenConnection(server: Server?) = mutex.withLock { + fun returnBrokenConnection(server: Server?) { if (server == null) { - return@withLock + return } logger?.debug("Returning broken connection: $server") - if (servers[nextServer % servers.count()] == server) { - nextServer++ + val servers = servers.get() + val currentIndex = nextServer.get() + + if (servers.isNotEmpty() && servers[currentIndex % servers.size] == server) { + nextServer.incrementAndGet() // TODO: (SK) Add server to ContentServerPenalty } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 481a6a6a..f909e123 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -125,22 +125,30 @@ class DepotDownloader @JvmOverloads constructor( val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" } + private val activeDownloads = AtomicInteger(0) + private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } - private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) - private val activeDownloads = AtomicInteger(0) - private val listeners = CopyOnWriteArrayList() - private var logger: Logger? = null - private var processingChannel = Channel(Channel.UNLIMITED) + private val httpClient: HttpClient by lazy { HttpClient(maxConnections = maxDownloads) } + private val lastFileProgressUpdate = ConcurrentHashMap() + + private val listeners = CopyOnWriteArrayList() + private val progressUpdateInterval = 500L // ms - private var steam3: Steam3Session? = null + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) private var cdnClientPool: CDNClientPool? = null private var config: Config = Config(androidEmulation = androidEmulation) + private var logger: Logger? = null + + private var processingChannel = Channel(Channel.UNLIMITED) + + private var steam3: Steam3Session? = null + // region [REGION] Private data classes. private data class NetworkChunkItem( @@ -189,7 +197,7 @@ class DepotDownloader @JvmOverloads constructor( // region [REGION] Downloading Operations @Throws(IllegalStateException::class) - suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { + private suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { val details = requireNotNull( steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) ) { "Pub File Null" } @@ -212,7 +220,7 @@ class DepotDownloader @JvmOverloads constructor( } } - suspend fun downloadUGC( + private suspend fun downloadUGC( appId: Int, ugcId: Long, ) { @@ -245,7 +253,7 @@ class DepotDownloader @JvmOverloads constructor( } @Throws(IllegalStateException::class, IOException::class) - suspend fun downloadWebFile(appId: Int, fileName: String, url: String) { + private suspend fun downloadWebFile(appId: Int, fileName: String, url: String) { val (success, installDir) = createDirectories(appId, 0, appId) if (!success) { @@ -260,7 +268,7 @@ class DepotDownloader @JvmOverloads constructor( filesystem.createDirectories(fileFinalPath.parent!!) filesystem.createDirectories(fileStagingPath.parent!!) - HttpClient.getClient(maxDownloads).use { client -> + httpClient.getClient().use { client -> logger?.debug("Starting download of $fileName...") val response = client.get(url) @@ -305,7 +313,7 @@ class DepotDownloader @JvmOverloads constructor( // L4D2 (app) supports LV @Throws(IllegalStateException::class) - suspend fun downloadApp( + private suspend fun downloadApp( appId: Int, depotManifestIds: List>, branch: String, @@ -1210,6 +1218,8 @@ class DepotDownloader @JvmOverloads constructor( val matchingChunks = arrayListOf() file.chunks.forEach { chunk -> + ensureActive() + val oldChunk = oldManifestFile.chunks.firstOrNull { c -> c.chunkID.contentEquals(chunk.chunkID) } @@ -1227,6 +1237,8 @@ class DepotDownloader @JvmOverloads constructor( filesystem.openReadOnly(fileFinalPath).use { handle -> orderedChunks.forEach { match -> + ensureActive() + // Read the chunk data into a byte array val length = match.oldChunk.uncompressedLength val buffer = ByteArray(length) @@ -1266,6 +1278,8 @@ class DepotDownloader @JvmOverloads constructor( } for (match in copyChunks) { + ensureActive() + val tmp = ByteArray(match.oldChunk.uncompressedLength) oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) @@ -1383,7 +1397,8 @@ class DepotDownloader @JvmOverloads constructor( var connection: Server? = null try { - connection = cdnClientPool!!.getConnection() + connection = cdnClientPool?.getConnection() + ?: throw IllegalStateException("ContentDownloader already closed") var cdnToken: String? = null @@ -1566,9 +1581,14 @@ class DepotDownloader @JvmOverloads constructor( */ fun add(item: DownloadItem) { runBlocking { - processingChannel.send(item) - activeDownloads.incrementAndGet() - notifyListeners { it.onItemAdded(item.appId) } + try { + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item) } + } catch (e: Exception) { + logger?.error(e) + throw e + } } } @@ -1577,10 +1597,15 @@ class DepotDownloader @JvmOverloads constructor( */ fun addAll(items: List) { runBlocking { - items.forEach { item -> - processingChannel.send(item) - activeDownloads.incrementAndGet() - notifyListeners { it.onItemAdded(item.appId) } + try { + items.forEach { item -> + processingChannel.send(item) + activeDownloads.incrementAndGet() + notifyListeners { it.onItemAdded(item) } + } + } catch (e: Exception) { + logger?.error(e) + throw e } } } @@ -1622,22 +1647,14 @@ class DepotDownloader @JvmOverloads constructor( when (item) { is PubFileItem -> { - if (item.pubfile == INVALID_MANIFEST_ID) { - logger?.debug("Invalid Pub File ID for ${item.appId}") - continue - } logger?.debug("Downloading PUB File for ${item.appId}") - notifyListeners { it.onDownloadStarted(item.appId) } + notifyListeners { it.onDownloadStarted(item) } downloadPubFile(item.appId, item.pubfile) } is UgcItem -> { - if (item.ugcId == INVALID_MANIFEST_ID) { - logger?.debug("Invalid UGC ID for ${item.appId}") - continue - } logger?.debug("Downloading UGC File for ${item.appId}") - notifyListeners { it.onDownloadStarted(item.appId) } + notifyListeners { it.onDownloadStarted(item) } downloadUGC(item.appId, item.ugcId) } @@ -1689,7 +1706,7 @@ class DepotDownloader @JvmOverloads constructor( } logger?.debug("Downloading App for ${item.appId}") - notifyListeners { it.onDownloadStarted(item.appId) } + notifyListeners { it.onDownloadStarted(item) } downloadApp( appId = item.appId, depotManifestIds = depotManifestIds, @@ -1703,10 +1720,10 @@ class DepotDownloader @JvmOverloads constructor( } } - notifyListeners { it.onDownloadCompleted(item.appId) } + notifyListeners { it.onDownloadCompleted(item) } } catch (e: Exception) { logger?.error("Error downloading item ${item.appId}: ${e.message}", e) - notifyListeners { it.onDownloadFailed(item.appId, e) } + notifyListeners { it.onDownloadFailed(item, e) } } finally { activeDownloads.decrementAndGet() } @@ -1718,7 +1735,7 @@ class DepotDownloader @JvmOverloads constructor( scope.cancel("DepotDownloader Closing") - HttpClient.close() + httpClient.close() lastFileProgressUpdate.clear() listeners.clear() diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt index 75e4f612..67f35d48 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/HttpClient.kt @@ -6,21 +6,36 @@ import io.ktor.client.engine.cio.CIO import io.ktor.client.engine.cio.endpoint import io.ktor.client.plugins.UserAgent import kotlinx.coroutines.isActive +import java.io.Closeable /** - * Singleton HTTP client for content downloader operations. - * Provides a shared, configured Ktor HTTP client optimized for Steam CDN downloads. - * The client is lazily initialized on first use and reused across all download operations. - * Connection pooling and timeouts are configured based on the maximum concurrent downloads. + * HTTP client wrapper for content downloader operations. + * + * Provides a configured Ktor HTTP client optimized for Steam CDN downloads. + * Each instance maintains its own connection pool based on the specified + * maximum concurrent connections. + * + * @param maxConnections Maximum number of concurrent connections * * @author Lossy * @since Oct 1, 2025 */ -object HttpClient { +class HttpClient( + private val maxConnections: Int, +) : Closeable { private var httpClient: HttpClient? = null - fun getClient(maxConnections: Int = 8): HttpClient { + /** + * Returns the HTTP client instance, creating it lazily on first access. + * + * The client is configured with: + * - Custom User-Agent identifying JavaSteam DepotDownloader + * - Connection pooling based on [maxConnections] + * - 5 second keep-alive and connect timeout + * - 30 second request timeout + */ + fun getClient(): HttpClient { if (httpClient?.isActive != true) { httpClient = HttpClient(CIO) { install(UserAgent) { @@ -42,7 +57,7 @@ object HttpClient { return httpClient!! } - fun close() { + override fun close() { if (httpClient?.isActive == true) { httpClient?.close() httpClient = null diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt index 9b4ab7f2..072ea85a 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -1,6 +1,7 @@ package `in`.dragonbra.javasteam.depotdownloader import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress +import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress @@ -17,31 +18,31 @@ interface IDownloadListener { /** * Called when an item is added to the download queue. * - * @param appId The application ID that was queued + * @param item The [DownloadItem] that was queued */ - fun onItemAdded(appId: Int) {} + fun onItemAdded(item: DownloadItem) {} /** * Called when a download begins processing. * - * @param appId The application ID being downloaded + * @param item The [DownloadItem] being downloaded */ - fun onDownloadStarted(appId: Int) {} + fun onDownloadStarted(item: DownloadItem) {} /** * Called when a download completes successfully. * - * @param appId The application ID that finished downloading + * @param item The [DownloadItem] that finished downloading */ - fun onDownloadCompleted(appId: Int) {} + fun onDownloadCompleted(item: DownloadItem) {} /** * Called when a download fails with an error. * - * @param appId The application ID that failed + * @param item The [DownloadItem] that failed * @param error The exception that caused the failure */ - fun onDownloadFailed(appId: Int, error: Throwable) {} + fun onDownloadFailed(item: DownloadItem, error: Throwable) {} /** * Called periodically with overall download progress across all items. diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt index e94ecdd6..1c6baa00 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -2,6 +2,8 @@ package `in`.dragonbra.javasteam.depotdownloader.data // https://kotlinlang.org/docs/coding-conventions.html#source-file-organization +// TODO should these be a builder pattern for Java users? + /** * Base class for downloadable Steam content items. * @property appId The Steam application ID @@ -16,6 +18,7 @@ abstract class DownloadItem( val appId: Int, val installDirectory: String?, val installToGameNameDirectory: Boolean, + val verify: Boolean, // TODO val downloadManifestOnly: Boolean, ) @@ -32,8 +35,9 @@ class UgcItem @JvmOverloads constructor( val ugcId: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, + verify: Boolean = false, downloadManifestOnly: Boolean = false, -) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) /** * Represents a Steam published file for download. @@ -48,8 +52,9 @@ class PubFileItem @JvmOverloads constructor( val pubfile: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, + verify: Boolean = false, downloadManifestOnly: Boolean = false, -) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) /** * Represents a Steam application/game for download from a depot. @@ -84,5 +89,6 @@ class AppItem @JvmOverloads constructor( val lowViolence: Boolean = false, val depot: List = emptyList(), val manifest: List = emptyList(), + verify: Boolean = false, downloadManifestOnly: Boolean = false, -) : DownloadItem(appId, installDirectory, installToGameNameDirectory, downloadManifestOnly) +) : DownloadItem(appId, installDirectory, installToGameNameDirectory, verify, downloadManifestOnly) diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index 9d25aaf1..bcb28c2b 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -146,7 +146,7 @@ private void onConnected(ConnectedCallback callback) { accountName = pollResponse.getAccountName(); refreshToken = pollResponse.getRefreshToken(); - // Save out refresh token for automatic login on next sample run. + // Save our refresh token for automatic login on next sample run. Files.writeString(path, pollResponse.getRefreshToken()); } else { System.out.println("Existing refresh token found"); @@ -251,6 +251,7 @@ private void downloadApp() { /* pubfile */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, + /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false ); // TODO find actual pub item @@ -259,6 +260,7 @@ private void downloadApp() { /* ugcId */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, + /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false ); // TODO find actual ugc item @@ -277,6 +279,7 @@ private void downloadApp() { /* (Optional) lowViolence */ false, /* (Optional) depot */ List.of(), /* (Optional) manifest */ List.of(), + /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false ); @@ -287,7 +290,7 @@ private void downloadApp() { // After 'depotDownloader' is constructed, items added are downloaded in a First-In, First-Out queue on the fly. // Add a singular item to process. - depotDownloader.add(new AppItem(appId, true)); + depotDownloader.add(new AppItem(appId, true, "steamapps")); // You can add a List of items to be processed. // depotDownloader.add(List.of()); @@ -312,72 +315,93 @@ private void downloadApp() { // Depot Downloader Callbacks. @Override - public void onItemAdded(int appId) { - System.out.println("Depot Downloader: Item Added: " + appId); - System.out.println(" ---- "); + public void onItemAdded(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Item Added: " + item.getAppId() + "\n ---- "); } @Override - public void onDownloadStarted(int appId) { - System.out.println("Depot Downloader: Download started for item: " + appId); - System.out.println(" ---- "); + public void onDownloadStarted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download started for item: " + item.getAppId() + "\n ---- "); } @Override - public void onDownloadCompleted(int appId) { - System.out.println("Depot Downloader: Download completed for item: " + appId); - System.out.println(" ---- "); + public void onDownloadCompleted(@NotNull DownloadItem item) { + System.out.println("Depot Downloader: Download completed for item: " + item.getAppId() + "\n ---- "); } @Override - public void onDownloadFailed(int appId, @NotNull Throwable error) { - System.out.println("Depot Downloader: Download failed for item: " + appId); - System.err.println(error.getMessage()); - System.out.println(" ---- "); + public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { + System.out.println("Depot Downloader: Download failed for item: " + item.getAppId() + "\n ---- "); + if (!error.getMessage().isEmpty()) { + System.err.println(error.getMessage()); + } } @Override public void onOverallProgress(@NotNull OverallProgress progress) { - System.out.println("Depot Downloader: Overall Progress"); - System.out.println("currentItem: " + progress.getCurrentItem()); - System.out.println("totalItems: " + progress.getTotalItems()); - System.out.println("totalBytesDownloaded: " + progress.getTotalBytesDownloaded()); - System.out.println("totalBytesExpected: " + progress.getTotalBytesExpected()); - System.out.println("status: " + progress.getStatus()); - System.out.println("percentComplete: " + progress.getPercentComplete()); - System.out.println(" ---- "); + System.out.printf( + "Depot Downloader: Overall Progress\n" + + "currentItem: %d\n" + + "totalItems: %d\n" + + "totalBytesDownloaded: %d\n" + + "totalBytesExpected: %d\n" + + "status: %s\n" + + "percentComplete: %.2f\n ---- %n \n", + progress.getCurrentItem(), + progress.getTotalItems(), + progress.getTotalBytesDownloaded(), + progress.getTotalBytesExpected(), + progress.getStatus(), + progress.getPercentComplete() + ); } @Override public void onDepotProgress(@NotNull DepotProgress progress) { - System.out.println("Depot Downloader: Depot Progress"); - System.out.println("depotId: " + progress.getDepotId()); - System.out.println("filesCompleted: " + progress.getFilesCompleted()); - System.out.println("totalFiles: " + progress.getTotalFiles()); - System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); - System.out.println("totalBytes: " + progress.getTotalBytes()); - System.out.println("status: " + progress.getStatus()); - System.out.println("percentComplete: " + progress.getPercentComplete()); - System.out.println(" ---- "); + System.out.printf( + "Depot Downloader: Depot Progress\n" + + "depotId: %d\n" + + "filesCompleted: %d\n" + + "totalFiles: %d\n" + + "bytesDownloaded: %d\n" + + "totalBytes: %d\n" + + "status: %s\n" + + "percentComplete: %.2f\n ---- %n \n", + progress.getDepotId(), + progress.getFilesCompleted(), + progress.getTotalFiles(), + progress.getBytesDownloaded(), + progress.getTotalBytes(), + progress.getStatus(), + progress.getPercentComplete() + ); } @Override public void onFileProgress(@NotNull FileProgress progress) { - System.out.println("Depot Downloader: File Progress"); - System.out.println("depotId: " + progress.getDepotId()); - System.out.println("fileName: " + progress.getFileName()); - System.out.println("bytesDownloaded: " + progress.getBytesDownloaded()); - System.out.println("totalBytes: " + progress.getTotalBytes()); - System.out.println("chunksCompleted: " + progress.getChunksCompleted()); - System.out.println("totalChunks: " + progress.getTotalChunks()); - System.out.println("status: " + progress.getStatus()); - System.out.println("percentComplete: " + progress.getPercentComplete()); - System.out.println(" ---- "); + System.out.printf( + "Depot Downloader: File Progress\n" + + "depotId: %d\n" + + "fileName: %s\n" + + "bytesDownloaded: %d\n" + + "totalBytes: %d\n" + + "chunksCompleted: %d\n" + + "totalChunks: %d\n" + + "status: %s\n" + + "percentComplete: %.2f\n ---- %n \n", + progress.getDepotId(), + progress.getFileName(), + progress.getBytesDownloaded(), + progress.getTotalBytes(), + progress.getChunksCompleted(), + progress.getTotalChunks(), + progress.getStatus(), + progress.getPercentComplete() + ); } @Override public void onStatusUpdate(@NotNull String message) { - System.out.println("Depot Downloader: Status Message: " + message); - System.out.println(" ---- "); + System.out.println("Depot Downloader: Status Message: " + message + "\n ---- "); } } diff --git a/src/main/java/in/dragonbra/javasteam/types/FileData.kt b/src/main/java/in/dragonbra/javasteam/types/FileData.kt index 53a7c2f4..41ee3ead 100644 --- a/src/main/java/in/dragonbra/javasteam/types/FileData.kt +++ b/src/main/java/in/dragonbra/javasteam/types/FileData.kt @@ -17,7 +17,6 @@ import java.util.EnumSet * @param fileHash Gets SHA-1 hash of this file. * @param linkTarget Gets symlink target of this file. */ -@Suppress("ArrayInDataClass") data class FileData( var fileName: String = "", var fileNameHash: ByteArray = byteArrayOf(), @@ -48,4 +47,32 @@ data class FileData( fileHash = hash, linkTarget = linkTarget, ) + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as FileData + + if (totalSize != other.totalSize) return false + if (fileName != other.fileName) return false + if (!fileNameHash.contentEquals(other.fileNameHash)) return false + if (chunks != other.chunks) return false + if (flags != other.flags) return false + if (!fileHash.contentEquals(other.fileHash)) return false + if (linkTarget != other.linkTarget) return false + + return true + } + + override fun hashCode(): Int { + var result = totalSize.hashCode() + result = 31 * result + fileName.hashCode() + result = 31 * result + fileNameHash.contentHashCode() + result = 31 * result + chunks.hashCode() + result = 31 * result + flags.hashCode() + result = 31 * result + fileHash.contentHashCode() + result = 31 * result + (linkTarget?.hashCode() ?: 0) + return result + } } From 5e88d2416a5f7b733b04677152450415e0d8b2d2 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 11:25:55 -0500 Subject: [PATCH 10/44] Fix getting sha file --- .../main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt index 9d2bc32f..a46951a0 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Util.kt @@ -99,7 +99,7 @@ object Util { if (FileSystem.SYSTEM.exists(filename)) { val expectedChecksum = try { - FileSystem.SYSTEM.read(filename / ".sha") { + FileSystem.SYSTEM.read("$filename.sha".toPath()) { readByteArray() } } catch (e: IOException) { From 3dd1b115bb10ea77bd2feaf4c430e4ccef2608e6 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 14:54:38 -0500 Subject: [PATCH 11/44] Simplify IDownloadListener, fix up DepotDownloader yielding. Update SampleDownloadApp --- .../depotdownloader/DepotDownloader.kt | 444 ++++++++---------- .../depotdownloader/IDownloadListener.kt | 50 +- .../depotdownloader/Steam3Session.kt | 7 +- .../depotdownloader/data/DownloadItems.kt | 2 +- .../depotdownloader/data/DownloadProgress.kt | 111 ----- .../_023_downloadapp/SampleDownloadApp.java | 128 ++--- 6 files changed, 243 insertions(+), 499 deletions(-) delete mode 100644 javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index f909e123..415fec72 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -5,13 +5,9 @@ import `in`.dragonbra.javasteam.depotdownloader.data.ChunkMatch import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadCounter import `in`.dragonbra.javasteam.depotdownloader.data.DepotDownloadInfo import `in`.dragonbra.javasteam.depotdownloader.data.DepotFilesData -import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem -import `in`.dragonbra.javasteam.depotdownloader.data.DownloadStatus -import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress import `in`.dragonbra.javasteam.depotdownloader.data.FileStreamData import `in`.dragonbra.javasteam.depotdownloader.data.GlobalDownloadCounter -import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress import `in`.dragonbra.javasteam.depotdownloader.data.PubFileItem import `in`.dragonbra.javasteam.depotdownloader.data.UgcItem import `in`.dragonbra.javasteam.enums.EAccountType @@ -38,7 +34,6 @@ import io.ktor.client.request.get import io.ktor.client.statement.bodyAsChannel import io.ktor.http.HttpHeaders import io.ktor.utils.io.core.readAvailable -import io.ktor.utils.io.core.remaining import io.ktor.utils.io.readRemaining import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope @@ -50,7 +45,6 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay import kotlinx.coroutines.ensureActive import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking @@ -125,14 +119,12 @@ class DepotDownloader @JvmOverloads constructor( val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" } - private val activeDownloads = AtomicInteger(0) + private val completionFuture = CompletableFuture() private val filesystem: FileSystem by lazy { FileSystem.SYSTEM } private val httpClient: HttpClient by lazy { HttpClient(maxConnections = maxDownloads) } - private val lastFileProgressUpdate = ConcurrentHashMap() - private val listeners = CopyOnWriteArrayList() private val progressUpdateInterval = 500L // ms @@ -280,17 +272,16 @@ class DepotDownloader @JvmOverloads constructor( filesystem.sink(fileStagingPath).buffer().use { sink -> val buffer = Buffer() + val tempArray = ByteArray(DEFAULT_BUFFER_SIZE) + while (!channel.isClosedForRead) { val packet = channel.readRemaining(DEFAULT_BUFFER_SIZE.toLong()) if (!packet.exhausted()) { - // Read from Ktor packet into Okio buffer - val bytesRead = packet.remaining.toInt() - val tempArray = ByteArray(bytesRead) - packet.readAvailable(tempArray) - buffer.write(tempArray) - - // Write from buffer to sink - sink.writeAll(buffer) + val bytesRead = packet.readAvailable(tempArray, 0, tempArray.size) + if (bytesRead > 0) { + buffer.write(tempArray, 0, bytesRead) + sink.writeAll(buffer) + } } } } @@ -368,14 +359,11 @@ class DepotDownloader @JvmOverloads constructor( logger?.debug("Using app branch: $branch") depots?.children?.forEach { depotSection -> - @Suppress("VariableInitializerIsRedundant") - var id = INVALID_DEPOT_ID - if (depotSection.children.isEmpty()) { return@forEach } - id = depotSection.name?.toIntOrNull() ?: return@forEach + val id: Int = depotSection.name?.toIntOrNull() ?: return@forEach if (hasSpecificDepots && !depotIdsExpected.contains(id)) { return@forEach @@ -780,15 +768,13 @@ class DepotDownloader @JvmOverloads constructor( val depotsToDownload = ArrayList(depots.size) val allFileNamesAllDepots = hashSetOf() - var completedDepots = 0 - // First, fetch all the manifests for each depot (including previous manifests) and perform the initial setup depots.forEach { depot -> val depotFileData = processDepotManifestAndFiles(depot, downloadCounter) if (depotFileData != null) { depotsToDownload.add(depotFileData) - allFileNamesAllDepots.union(depotFileData.allFileNames) + allFileNamesAllDepots.addAll(depotFileData.allFileNames) } ensureActive() @@ -807,22 +793,6 @@ class DepotDownloader @JvmOverloads constructor( depotsToDownload.forEach { depotFileData -> downloadSteam3DepotFiles(downloadCounter, depotFileData, allFileNamesAllDepots) - - completedDepots++ - - val snapshot = synchronized(downloadCounter) { - OverallProgress( - currentItem = completedDepots, - totalItems = depotsToDownload.size, - totalBytesDownloaded = downloadCounter.totalBytesUncompressed, - totalBytesExpected = downloadCounter.completeDownloadSize, - status = DownloadStatus.DOWNLOADING - ) - } - - notifyListeners { listener -> - listener.onOverallProgress(progress = snapshot) - } } logger?.debug( @@ -993,7 +963,11 @@ class DepotDownloader @JvmOverloads constructor( val allFileNames = HashSet(filesAfterExclusions.size) // Pre-process - filesAfterExclusions.forEach { file -> + filesAfterExclusions.forEachIndexed { index, file -> + if (index % 50 == 0) { + ensureActive() // Check cancellation periodically + } + allFileNames.add(file.fileName) val fileFinalPath = depot.installDir / file.fileName @@ -1038,89 +1012,40 @@ class DepotDownloader @JvmOverloads constructor( val networkChunkQueue = Channel(Channel.UNLIMITED) try { - val filesCompleted = AtomicInteger(0) - val lastReportedProgress = AtomicInteger(0) coroutineScope { // First parallel loop - process files and enqueue chunks - files.map { file -> - async { - yield() // Does this matter if its before? - downloadSteam3DepotFile( - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = file, - networkChunkQueue = networkChunkQueue - ) - - val completed = filesCompleted.incrementAndGet() - if (completed % 10 == 0 || completed == files.size) { - val snapshot = synchronized(depotCounter) { - DepotProgress( - depotId = depot.depotId, - filesCompleted = completed, - totalFiles = files.size, - bytesDownloaded = depotCounter.sizeDownloaded, - totalBytes = depotCounter.completeDownloadSize, - status = DownloadStatus.PREPARING // Changed from DOWNLOADING - ) - } + files.chunked(50).forEach { batch -> + yield() - val lastReported = lastReportedProgress.get() - if (completed > lastReported && - lastReportedProgress.compareAndSet( - lastReported, - completed - ) - ) { - notifyListeners { listener -> - listener.onDepotProgress(snapshot) - } - } + batch.map { file -> + async { + downloadSteam3DepotFile( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = file, + networkChunkQueue = networkChunkQueue + ) } - } - }.awaitAll() + }.awaitAll() + } // Close the channel to signal no more items will be added networkChunkQueue.close() - // After all files allocated, send one update showing preparation complete - val progressReporter = launch { - while (true) { - delay(1000) - val snapshot = synchronized(depotCounter) { - DepotProgress( - depotId = depot.depotId, - filesCompleted = files.size, - totalFiles = files.size, - bytesDownloaded = depotCounter.sizeDownloaded, - totalBytes = depotCounter.completeDownloadSize, - status = DownloadStatus.DOWNLOADING + // Second parallel loop - process chunks from queue + List(maxDownloads) { + async { + for (item in networkChunkQueue) { + downloadSteam3DepotFileChunk( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, + file = item.fileData, + fileStreamData = item.fileStreamData, + chunk = item.chunk ) } - notifyListeners { listener -> - listener.onDepotProgress(snapshot) - } } - } - - // Second parallel loop - process chunks from queue - try { - List(maxDownloads) { - async { - for (item in networkChunkQueue) { - downloadSteam3DepotFileChunk( - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = item.fileData, - fileStreamData = item.fileStreamData, - chunk = item.chunk - ) - } - } - }.awaitAll() - } finally { - progressReporter.cancel() - } + }.awaitAll() } } finally { if (!networkChunkQueue.isClosedForSend) { @@ -1158,6 +1083,16 @@ class DepotDownloader @JvmOverloads constructor( DepotConfigStore.getInstance().installedManifestIDs[depot.depotId] = depot.manifestId DepotConfigStore.save() + + // Notify depot completion + notifyListeners { listener -> + listener.onDepotCompleted( + depotId = depot.depotId, + compressedBytes = depotCounter.depotBytesCompressed, + uncompressedBytes = depotCounter.depotBytesUncompressed + ) + } + logger?.debug("Depot ${depot.depotId} - Downloaded ${depotCounter.depotBytesCompressed} bytes (${depotCounter.depotBytesUncompressed} bytes uncompressed)") } @@ -1218,7 +1153,7 @@ class DepotDownloader @JvmOverloads constructor( val matchingChunks = arrayListOf() file.chunks.forEach { chunk -> - ensureActive() + yield() val oldChunk = oldManifestFile.chunks.firstOrNull { c -> c.chunkID.contentEquals(chunk.chunkID) @@ -1237,7 +1172,7 @@ class DepotDownloader @JvmOverloads constructor( filesystem.openReadOnly(fileFinalPath).use { handle -> orderedChunks.forEach { match -> - ensureActive() + yield() // Read the chunk data into a byte array val length = match.oldChunk.uncompressedLength @@ -1266,28 +1201,24 @@ class DepotDownloader @JvmOverloads constructor( if (!hashMatches || neededChunks.isNotEmpty()) { filesystem.atomicMove(fileFinalPath, fileStagingPath) - try { - filesystem.openReadOnly(fileStagingPath).use { oldHandle -> - filesystem.openReadWrite(fileFinalPath).use { newHandle -> - try { - newHandle.resize(file.totalSize) - } catch (ex: IOException) { - throw DepotDownloaderException( - "Failed to resize file to expected size $fileFinalPath: ${ex.message}" - ) - } - - for (match in copyChunks) { - ensureActive() - - val tmp = ByteArray(match.oldChunk.uncompressedLength) - oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) - newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) - } + filesystem.openReadOnly(fileStagingPath).use { oldHandle -> + filesystem.openReadWrite(fileFinalPath).use { newHandle -> + try { + newHandle.resize(file.totalSize) + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to resize file to expected size $fileFinalPath: ${ex.message}" + ) + } + + for (match in copyChunks) { + ensureActive() + + val tmp = ByteArray(match.oldChunk.uncompressedLength) + oldHandle.read(match.oldChunk.offset, tmp, 0, tmp.size) + newHandle.write(match.newChunk.offset, tmp, 0, tmp.size) } } - } catch (e: Exception) { - logger?.error(e) } filesystem.delete(fileStagingPath) @@ -1390,149 +1321,137 @@ class DepotDownloader @JvmOverloads constructor( var written = 0 val chunkBuffer = ByteArray(chunk.uncompressedLength) - try { - do { - ensureActive() + do { + ensureActive() - var connection: Server? = null + var connection: Server? = null - try { - connection = cdnClientPool?.getConnection() - ?: throw IllegalStateException("ContentDownloader already closed") + try { + connection = cdnClientPool?.getConnection() + ?: throw IllegalStateException("ContentDownloader already closed") - var cdnToken: String? = null + var cdnToken: String? = null - val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] - if (authTokenCallbackPromise != null) { - try { - val result = authTokenCallbackPromise.await() - cdnToken = result.token - } catch (e: Exception) { - logger?.error("Failed to get CDN auth token: ${e.message}") - } + val authTokenCallbackPromise = steam3!!.cdnAuthTokens[depot.depotId to connection.host] + if (authTokenCallbackPromise != null) { + try { + val result = authTokenCallbackPromise.await() + cdnToken = result.token + } catch (e: Exception) { + logger?.error("Failed to get CDN auth token: ${e.message}") } + } - logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") - - written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( - depotId = depot.depotId, - chunk = chunk, - server = connection, - destination = chunkBuffer, - depotKey = depot.depotKey, - proxyServer = cdnClientPool!!.proxyServer, - cdnAuthToken = cdnToken, - ) + logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") - cdnClientPool!!.returnConnection(connection) + written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( + depotId = depot.depotId, + chunk = chunk, + server = connection, + destination = chunkBuffer, + depotKey = depot.depotKey, + proxyServer = cdnClientPool!!.proxyServer, + cdnAuthToken = cdnToken, + ) - break - } catch (e: CancellationException) { - logger?.error(e) - } catch (e: SteamKitWebRequestException) { - // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, - // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above - if (e.statusCode == 403 && - ( - !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host) || - steam3!!.cdnAuthTokens[depot.depotId to connection.host]?.isCompleted == false - ) - ) { - steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) + cdnClientPool!!.returnConnection(connection) - cdnClientPool!!.returnConnection(connection) + break + } catch (e: CancellationException) { + logger?.error(e) + } catch (e: SteamKitWebRequestException) { + // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, + // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above + if (e.statusCode == 403 && + ( + !steam3!!.cdnAuthTokens.containsKey(depot.depotId to connection!!.host) || + steam3!!.cdnAuthTokens[depot.depotId to connection.host]?.isCompleted == false + ) + ) { + steam3!!.requestCDNAuthToken(depot.appId, depot.depotId, connection) - continue - } + cdnClientPool!!.returnConnection(connection) - cdnClientPool!!.returnBrokenConnection(connection) + continue + } - // Unauthorized || Forbidden - if (e.statusCode == 401 || e.statusCode == 403) { - logger?.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") - break - } + cdnClientPool!!.returnBrokenConnection(connection) - logger?.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") - } catch (e: Exception) { - cdnClientPool!!.returnBrokenConnection(connection) - logger?.error("Encountered unexpected error downloading chunk $chunkID", e) + // Unauthorized || Forbidden + if (e.statusCode == 401 || e.statusCode == 403) { + logger?.error("Encountered ${e.statusCode} for chunk $chunkID. Aborting.") + break } - } while (written == 0) - if (written == 0) { - logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") - cancel() + logger?.error("Encountered error downloading chunk $chunkID: ${e.statusCode}") + } catch (e: Exception) { + cdnClientPool!!.returnBrokenConnection(connection) + logger?.error("Encountered unexpected error downloading chunk $chunkID", e) } + } while (written == 0) - // Throw the cancellation exception if requested so that this task is marked failed - ensureActive() + if (written == 0) { + logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") + cancel() + } - try { - fileStreamData.fileLock.lock() + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() - if (fileStreamData.fileHandle == null) { - val fileFinalPath = depot.installDir / file.fileName - fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) - } + try { + fileStreamData.fileLock.lock() - fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) - } finally { - fileStreamData.fileLock.unlock() + if (fileStreamData.fileHandle == null) { + val fileFinalPath = depot.installDir / file.fileName + fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) } + + fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) } finally { + fileStreamData.fileLock.unlock() } val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() if (remainingChunks == 0) { fileStreamData.fileHandle?.close() - } - - var sizeDownloaded = 0L - synchronized(depotDownloadCounter) { - sizeDownloaded = depotDownloadCounter.sizeDownloaded + written.toLong() - depotDownloadCounter.sizeDownloaded = sizeDownloaded - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - val now = System.currentTimeMillis() - val fileKey = "${depot.depotId}:${file.fileName}" - val lastUpdate = lastFileProgressUpdate[fileKey] ?: 0L - - if (now - lastUpdate >= progressUpdateInterval || remainingChunks == 0) { - lastFileProgressUpdate[fileKey] = now + // File completed - notify with percentage + val sizeDownloaded = synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + depotDownloadCounter.sizeDownloaded + } - val totalChunks = file.chunks.size - val completedChunks = totalChunks - remainingChunks + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } - // Approximate bytes based on completion ratio - val approximateBytesDownloaded = (file.totalSize * completedChunks) / totalChunks + val fileFinalPath = depot.installDir / file.fileName + val depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) notifyListeners { listener -> - listener.onFileProgress( - progress = FileProgress( - depotId = depot.depotId, - fileName = file.fileName, - bytesDownloaded = approximateBytesDownloaded, - totalBytes = file.totalSize, - chunksCompleted = completedChunks, - totalChunks = totalChunks, - status = if (remainingChunks == 0) DownloadStatus.COMPLETED else DownloadStatus.DOWNLOADING - ) + listener.onFileCompleted( + depotId = depot.depotId, + fileName = fileFinalPath.toString(), + depotPercentComplete = depotPercentage ) } - } - if (remainingChunks == 0) { - val fileFinalPath = depot.installDir / file.fileName - val percentage = (sizeDownloaded / depotDownloadCounter.completeDownloadSize.toFloat()) * 100.0f - logger?.debug("%.2f%% %s".format(percentage, fileFinalPath)) + logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) + } else { + // Just update counters without notifying + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } } } @@ -1583,7 +1502,6 @@ class DepotDownloader @JvmOverloads constructor( runBlocking { try { processingChannel.send(item) - activeDownloads.incrementAndGet() notifyListeners { it.onItemAdded(item) } } catch (e: Exception) { logger?.error(e) @@ -1600,7 +1518,6 @@ class DepotDownloader @JvmOverloads constructor( try { items.forEach { item -> processingChannel.send(item) - activeDownloads.incrementAndGet() notifyListeners { it.onItemAdded(item) } } } catch (e: Exception) { @@ -1611,14 +1528,15 @@ class DepotDownloader @JvmOverloads constructor( } /** - * Get the current queue size of pending items to be downloaded. + * Signals that no more items will be added to the download queue. + * After calling this, the downloader will complete once all queued items finish. + * + * This is called automatically by [close], but you can call it explicitly + * if you want to wait for completion without closing the downloader. */ - fun queueSize(): Int = activeDownloads.get() - - /** - * Get a boolean value if there are items in queue to be downloaded. - */ - fun isProcessing(): Boolean = activeDownloads.get() > 0 + fun finishAdding() { + processingChannel.close() + } // endregion @@ -1649,7 +1567,7 @@ class DepotDownloader @JvmOverloads constructor( is PubFileItem -> { logger?.debug("Downloading PUB File for ${item.appId}") notifyListeners { it.onDownloadStarted(item) } - downloadPubFile(item.appId, item.pubfile) + downloadPubFile(item.appId, item.pubFile) } is UgcItem -> { @@ -1724,10 +1642,25 @@ class DepotDownloader @JvmOverloads constructor( } catch (e: Exception) { logger?.error("Error downloading item ${item.appId}: ${e.message}", e) notifyListeners { it.onDownloadFailed(item, e) } - } finally { - activeDownloads.decrementAndGet() } } + + completionFuture.complete(null) + } + + /** + * Returns a CompletableFuture that completes when all queued downloads finish. + * @return CompletableFuture that completes when all downloads finish + */ + fun getCompletion(): CompletableFuture = completionFuture + + /** + * Blocks the current thread until all queued downloads complete. + * Convenience method that calls `getCompletion().join()`. + * @throws CompletionException if any download fails + */ + fun awaitCompletion() { + completionFuture.join() } override fun close() { @@ -1737,7 +1670,6 @@ class DepotDownloader @JvmOverloads constructor( httpClient.close() - lastFileProgressUpdate.clear() listeners.clear() steam3?.close() diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt index 072ea85a..331f4a72 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -1,9 +1,6 @@ package `in`.dragonbra.javasteam.depotdownloader -import `in`.dragonbra.javasteam.depotdownloader.data.DepotProgress import `in`.dragonbra.javasteam.depotdownloader.data.DownloadItem -import `in`.dragonbra.javasteam.depotdownloader.data.FileProgress -import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress /** * Listener interface for receiving download progress and status events. @@ -17,64 +14,47 @@ import `in`.dragonbra.javasteam.depotdownloader.data.OverallProgress interface IDownloadListener { /** * Called when an item is added to the download queue. - * - * @param item The [DownloadItem] that was queued */ fun onItemAdded(item: DownloadItem) {} /** * Called when a download begins processing. - * - * @param item The [DownloadItem] being downloaded */ fun onDownloadStarted(item: DownloadItem) {} /** * Called when a download completes successfully. - * - * @param item The [DownloadItem] that finished downloading */ fun onDownloadCompleted(item: DownloadItem) {} /** * Called when a download fails with an error. - * - * @param item The [DownloadItem] that failed - * @param error The exception that caused the failure */ fun onDownloadFailed(item: DownloadItem, error: Throwable) {} /** - * Called periodically with overall download progress across all items. - * Reports progress for the entire download queue, including completed - * and remaining items. - * - * @param progress Overall download statistics + * Called during file preparation with informational messages. + * Examples: "Pre-allocating depots\441\file.txt", "Validating file.cab" */ - fun onOverallProgress(progress: OverallProgress) {} - - /** - * Called periodically with progress for a specific depot. - * Reports file allocation and download progress for an individual depot. - * - * @param progress Depot-specific download statistics - */ - fun onDepotProgress(progress: DepotProgress) {} + fun onStatusUpdate(message: String) {} /** - * Called periodically with progress for a specific file. - * Reports chunk-level download progress for individual files. + * Called when a file completes downloading. + * Use this for printing progress like "20.42% depots\441\maps\ctf_haarp.bsp" * - * @param progress File-specific download statistics + * @param depotId The depot being downloaded + * @param fileName Relative file path + * @param depotPercentComplete Overall depot completion percentage (0f to 1f) */ - fun onFileProgress(progress: FileProgress) {} + fun onFileCompleted(depotId: Int, fileName: String, depotPercentComplete: Float) {} /** - * Called with informational status messages during download operations. - * Used for logging or displaying current operations like manifest - * downloads, file validation, and allocation. + * Called when a depot finishes downloading. + * Use this for printing summary like "Depot 228990 - Downloaded X bytes (Y bytes uncompressed)" * - * @param message Human-readable status message + * @param depotId The depot that completed + * @param compressedBytes Bytes transferred (compressed) + * @param uncompressedBytes Actual data size (uncompressed) */ - fun onStatusUpdate(message: String) {} + fun onDepotCompleted(depotId: Int, compressedBytes: Long, uncompressedBytes: Long) {} } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 9a3ebb94..1afcf241 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -152,6 +152,8 @@ class Steam3Session( private val packageInfoMutex = Mutex() suspend fun requestPackageInfo(packageIds: List) { packageInfoMutex.withLock { + logger?.debug("requestPackageInfo() invoked with ${packageIds.size} packageIds") + // I have a silly race condition??? val packages = packageIds.filter { !packageInfo.containsKey(it) } @@ -173,10 +175,7 @@ class Steam3Session( val packageInfoMultiple = steamApps!!.picsGetProductInfo(emptyList(), packageRequests).await() - logger?.debug( - "requestPackageInfo(packageIds =${packageIds.size}) \n" + - "picsGetProductInfo result size: ${packageInfoMultiple.results.size} " - ) + logger?.debug("requestPackageInfo() picsGetProductInfo result size: ${packageInfoMultiple.results.size} ") packageInfoMultiple.results.forEach { pkgInfo -> pkgInfo.packages.forEach { pkgValue -> diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt index 1c6baa00..813e0b52 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadItems.kt @@ -49,7 +49,7 @@ class UgcItem @JvmOverloads constructor( */ class PubFileItem @JvmOverloads constructor( appId: Int, - val pubfile: Long, + val pubFile: Long, installToGameNameDirectory: Boolean = false, installDirectory: String? = null, verify: Boolean = false, diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt deleted file mode 100644 index 2e058560..00000000 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/data/DownloadProgress.kt +++ /dev/null @@ -1,111 +0,0 @@ -package `in`.dragonbra.javasteam.depotdownloader.data - -/** - * Reports overall download progress across all queued items. - * Provides high-level statistics for the entire download session, tracking - * which item is currently processing and cumulative byte transfer. - * - * @property currentItem Number of items completed (1-based) - * @property totalItems Total number of items in the download session - * @property totalBytesDownloaded Cumulative uncompressed bytes downloaded across all depots - * @property totalBytesExpected Total uncompressed bytes expected for all items - * @property status Current download phase - * @property percentComplete Calculated completion percentage (0.0 to 100.0) - * - * @author Lossy - * @since Oct 1, 2025 - */ -data class OverallProgress( - val currentItem: Int, - val totalItems: Int, - val totalBytesDownloaded: Long, - val totalBytesExpected: Long, - val status: DownloadStatus, -) { - val percentComplete: Double - get() = if (totalBytesExpected > 0) { - (totalBytesDownloaded.toDouble() / totalBytesExpected) * 100.0 - } else { - 0.0 - } -} - -/** - * Reports download progress for a specific depot within an item. - * Tracks both file-level progress (allocation/validation) and byte-level - * download progress. During the [DownloadStatus.PREPARING] phase, tracks - * file allocation; during [DownloadStatus.DOWNLOADING], tracks actual transfers. - * - * @property depotId The Steam depot identifier - * @property filesCompleted Number of files fully allocated or downloaded - * @property totalFiles Total files to process in this depot (excludes directories) - * @property bytesDownloaded Uncompressed bytes successfully downloaded - * @property totalBytes Total uncompressed bytes expected for this depot - * @property status Current depot processing phase - * @property percentComplete Calculated completion percentage (0.0 to 100.0) - * - * @author Lossy - * @since Oct 1, 2025 - */ -data class DepotProgress( - val depotId: Int, - val filesCompleted: Int, - val totalFiles: Int, - val bytesDownloaded: Long, - val totalBytes: Long, - val status: DownloadStatus, -) { - val percentComplete: Double - get() = if (totalBytes > 0) { - (bytesDownloaded.toDouble() / totalBytes) * 100.0 - } else { - 0.0 - } -} - -/** - * Reports download progress for an individual file. - * Provides chunk-level granularity for tracking file downloads. Updates are - * throttled to every 500ms to avoid excessive callback overhead. - * - * @property depotId The Steam depot containing this file - * @property fileName Relative path of the file within the depot - * @property bytesDownloaded Approximate uncompressed bytes downloaded (based on chunk completion) - * @property totalBytes Total uncompressed file size - * @property chunksCompleted Number of chunks successfully downloaded and written - * @property totalChunks Total chunks comprising this file - * @property status Current file download status - * @property percentComplete Calculated completion percentage (0.0 to 100.0) - * - * @author Lossy - * @since Oct 1, 2025 - */ -data class FileProgress( - val depotId: Int, - val fileName: String, - val bytesDownloaded: Long, - val totalBytes: Long, - val chunksCompleted: Int, - val totalChunks: Int, - val status: DownloadStatus, -) { - val percentComplete: Double - get() = if (totalBytes > 0) { - (bytesDownloaded.toDouble() / totalBytes) * 100.0 - } else { - 0.0 - } -} - -/** - * Represents the current phase of a download operation. - * - * @property PREPARING File allocation and validation phase. Files are being pre-allocated on disk and existing content is being verified. - * @property DOWNLOADING Active chunk download phase. Content is being transferred from CDN. - * @property COMPLETED Download finished successfully. All files written and verified. - */ -enum class DownloadStatus { - PREPARING, - DOWNLOADING, - COMPLETED, -} diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index bcb28c2b..b6ec70b4 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -28,10 +28,8 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; -import java.util.Scanner; import java.util.concurrent.CancellationException; - /** * @author Oxters * @since 2024-11-07 @@ -245,27 +243,26 @@ private void downloadApp() { // Add this class as a listener of IDownloadListener depotDownloader.addListener(this); - // An app id is required at minimum for all item types. var pubItem = new PubFileItem( - /* appId */ 0, - /* pubfile */ 0, + /* (Required) appId */ 0, + /* (Required) pubFile */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false - ); // TODO find actual pub item + ); var ugcItem = new UgcItem( - /* appId */0, - /* ugcId */ 0, + /* (Required) appId */0, + /* (Required) ugcId */ 0, /* (Optional) installToGameNameDirectory */ false, /* (Optional) installDirectory */ null, /* (Optional) verify */ false, /* (Optional) downloadManifestOnly */ false - ); // TODO find actual ugc item + ); var appItem = new AppItem( - /* appId */ 204360, + /* (Required) appId */ 1303350, /* (Optional) installToGameNameDirectory */ true, /* (Optional) installDirectory */ "steamapps", /* (Optional) branch */ "public", @@ -283,26 +280,25 @@ private void downloadApp() { /* (Optional) downloadManifestOnly */ false ); - var scanner = new Scanner(System.in); - System.out.print("Enter a game app id: "); - var appId = scanner.nextInt(); - - // After 'depotDownloader' is constructed, items added are downloaded in a First-In, First-Out queue on the fly. + // Items added are downloaded automatically in a FIFO (First-In, First-Out) queue. // Add a singular item to process. - depotDownloader.add(new AppItem(appId, true, "steamapps")); + depotDownloader.add(appItem); // You can add a List of items to be processed. - // depotDownloader.add(List.of()); - - // Stay here while content downloads. Note this sample is synchronous so we'll loop here. - while (depotDownloader.isProcessing()) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - break; - } - } + // depotDownloader.add(List.of(a, b, c)); + + // Signal the downloader that no more items will be added. + // Once all items in queue are done, 'completion' will signal that everything had finished. + depotDownloader.finishAdding(); + + // Block until we're done downloading. + // Note: If you did not call `finishAdding()` before awaiting, depotDownloader will be expecting + // more items to be added to queue. It may look like a hang. You could call `close()` to finish too. + depotDownloader.awaitCompletion(); + + // Kotlin users can use: + // depotDownloader.getCompletion().await() // Remove this class as a listener of IDownloadListener depotDownloader.removeListener(this); @@ -316,92 +312,40 @@ private void downloadApp() { @Override public void onItemAdded(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Item Added: " + item.getAppId() + "\n ---- "); + System.out.println("Item " + item.getAppId() + " added to queue."); } @Override public void onDownloadStarted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download started for item: " + item.getAppId() + "\n ---- "); + System.out.println("Item " + item.getAppId() + " download started."); } @Override public void onDownloadCompleted(@NotNull DownloadItem item) { - System.out.println("Depot Downloader: Download completed for item: " + item.getAppId() + "\n ---- "); + System.out.println("Item " + item.getAppId() + " download completed."); } @Override public void onDownloadFailed(@NotNull DownloadItem item, @NotNull Throwable error) { - System.out.println("Depot Downloader: Download failed for item: " + item.getAppId() + "\n ---- "); - if (!error.getMessage().isEmpty()) { - System.err.println(error.getMessage()); - } + System.out.println("Item " + item.getAppId() + " failed to download"); + System.err.println(error.getMessage()); } @Override - public void onOverallProgress(@NotNull OverallProgress progress) { - System.out.printf( - "Depot Downloader: Overall Progress\n" + - "currentItem: %d\n" + - "totalItems: %d\n" + - "totalBytesDownloaded: %d\n" + - "totalBytesExpected: %d\n" + - "status: %s\n" + - "percentComplete: %.2f\n ---- %n \n", - progress.getCurrentItem(), - progress.getTotalItems(), - progress.getTotalBytesDownloaded(), - progress.getTotalBytesExpected(), - progress.getStatus(), - progress.getPercentComplete() - ); - } - - @Override - public void onDepotProgress(@NotNull DepotProgress progress) { - System.out.printf( - "Depot Downloader: Depot Progress\n" + - "depotId: %d\n" + - "filesCompleted: %d\n" + - "totalFiles: %d\n" + - "bytesDownloaded: %d\n" + - "totalBytes: %d\n" + - "status: %s\n" + - "percentComplete: %.2f\n ---- %n \n", - progress.getDepotId(), - progress.getFilesCompleted(), - progress.getTotalFiles(), - progress.getBytesDownloaded(), - progress.getTotalBytes(), - progress.getStatus(), - progress.getPercentComplete() - ); + public void onStatusUpdate(@NotNull String message) { + System.out.println("Status: " + message); } @Override - public void onFileProgress(@NotNull FileProgress progress) { - System.out.printf( - "Depot Downloader: File Progress\n" + - "depotId: %d\n" + - "fileName: %s\n" + - "bytesDownloaded: %d\n" + - "totalBytes: %d\n" + - "chunksCompleted: %d\n" + - "totalChunks: %d\n" + - "status: %s\n" + - "percentComplete: %.2f\n ---- %n \n", - progress.getDepotId(), - progress.getFileName(), - progress.getBytesDownloaded(), - progress.getTotalBytes(), - progress.getChunksCompleted(), - progress.getTotalChunks(), - progress.getStatus(), - progress.getPercentComplete() - ); + public void onFileCompleted(int depotId, @NotNull String fileName, float depotPercentComplete) { + var complete = String.format("%.2f%%", depotPercentComplete * 100f); + System.out.println("Depot " + depotId + " with file " + fileName + " completed. " + complete); } @Override - public void onStatusUpdate(@NotNull String message) { - System.out.println("Depot Downloader: Status Message: " + message + "\n ---- "); + public void onDepotCompleted(int depotId, long compressedBytes, long uncompressedBytes) { + System.out.println("Depot " + depotId + " completed."); + System.out.println("\t" + compressedBytes + " compressed bytes"); + System.out.println("\t" + uncompressedBytes + " uncompressed bytes"); } } From 2a92260a5fb9b1988b315fb8d602a03dd6c48f47 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 15:03:54 -0500 Subject: [PATCH 12/44] Bump version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index fcc9e504..84e27e9e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -20,7 +20,7 @@ plugins { allprojects { group = "in.dragonbra" - version = "1.7.1-SNAPSHOT" + version = "1.8.0-SNAPSHOT" } repositories { From e0c2515f3c5ed190b3f9616c82b388212ff38af3 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 13 Oct 2025 16:21:24 -0500 Subject: [PATCH 13/44] Add reified method for removeHandler. --- .../dragonbra/javasteam/steam/steamclient/SteamClient.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt index 9ea0928b..061e7789 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt @@ -122,6 +122,15 @@ class SteamClient @JvmOverloads constructor( handlers.remove(handler) } + /** + * Kotlin Helper: + * Removes a registered handler by name. + * @param T The handler name to remove. + */ + inline fun removeHandler() { + removeHandler(T::class.java) + } + /** * Removes a registered handler. * @param handler The handler name to remove. From 8cfab8a813ee95ce58298b0443c7d1491107eeea Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Tue, 14 Oct 2025 16:40:04 -0500 Subject: [PATCH 14/44] Add reified method for addHandler. --- .../javasteam/steam/steamclient/SteamClient.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt index 061e7789..f4ab15dc 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt @@ -109,6 +109,16 @@ class SteamClient @JvmOverloads constructor( addHandlerCore(handler) } + /** + * Kotlin Helper: + * Adds a new handler to the internal list of message handlers. + * @param T The handler to add. + */ + inline fun addHandler() { + val handler = T::class.java.getDeclaredConstructor().newInstance() + addHandler(handler) + } + private fun addHandlerCore(handler: ClientMsgHandler) { handler.setup(this) handlers[handler.javaClass] = handler From 82e25f0af1e948aeafc01d9704269c49e5c02072 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 14 Nov 2025 10:47:34 -0600 Subject: [PATCH 15/44] Fix OOM on android when allocating a file. --- .../depotdownloader/DepotDownloader.kt | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 415fec72..39295db4 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -59,6 +59,7 @@ import okio.buffer import org.apache.commons.lang3.SystemUtils import java.io.Closeable import java.io.IOException +import java.io.RandomAccessFile import java.lang.IllegalStateException import java.time.Instant import java.time.temporal.ChronoUnit @@ -1130,8 +1131,9 @@ class DepotDownloader @JvmOverloads constructor( // create new file. need all chunks try { - filesystem.openReadWrite(fileFinalPath).use { handle -> - handle.resize(file.totalSize) + // okio resize can OOM for large files on android. + RandomAccessFile(fileFinalPath.toFile(), "rw").use { + it.setLength(file.totalSize) } } catch (e: IOException) { throw DepotDownloaderException("Failed to allocate file $fileFinalPath: ${e.message}") @@ -1201,16 +1203,19 @@ class DepotDownloader @JvmOverloads constructor( if (!hashMatches || neededChunks.isNotEmpty()) { filesystem.atomicMove(fileFinalPath, fileStagingPath) + try { + RandomAccessFile(fileFinalPath.toFile(), "rw").use { raf -> + raf.setLength(file.totalSize) + } + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to resize file to expected size $fileFinalPath: ${ex.message}" + ) + } + filesystem.openReadOnly(fileStagingPath).use { oldHandle -> filesystem.openReadWrite(fileFinalPath).use { newHandle -> - try { - newHandle.resize(file.totalSize) - } catch (ex: IOException) { - throw DepotDownloaderException( - "Failed to resize file to expected size $fileFinalPath: ${ex.message}" - ) - } - + // okio resize can OOM for large files on android. for (match in copyChunks) { ensureActive() @@ -1226,18 +1231,21 @@ class DepotDownloader @JvmOverloads constructor( } } else { // No old manifest or file not in old manifest. We must validate. - filesystem.openReadWrite(fileFinalPath).use { handle -> - val fileSize = filesystem.metadata(fileFinalPath).size ?: 0L - if (fileSize.toULong() != file.totalSize.toULong()) { - try { - handle.resize(file.totalSize) - } catch (ex: IOException) { - throw DepotDownloaderException( - "Failed to allocate file $fileFinalPath: ${ex.message}" - ) + val fileSize = filesystem.metadata(fileFinalPath).size ?: 0L + if (fileSize.toULong() != file.totalSize.toULong()) { + try { + // okio resize can OOM for large files on android. + RandomAccessFile(fileFinalPath.toFile(), "rw").use { raf -> + raf.setLength(file.totalSize) } + } catch (ex: IOException) { + throw DepotDownloaderException( + "Failed to allocate file $fileFinalPath: ${ex.message}" + ) } + } + filesystem.openReadWrite(fileFinalPath).use { handle -> logger?.debug("Validating $fileFinalPath") notifyListeners { it.onStatusUpdate("Validating: ${file.fileName}") } From a4fd3238d62244e06341d677c8e612071ae4ed10 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 14 Nov 2025 11:16:24 -0600 Subject: [PATCH 16/44] Allow getting more servers in getServersForSteamPipe --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 39295db4..e98cd1c2 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -763,7 +763,8 @@ class DepotDownloader @JvmOverloads constructor( } private suspend fun downloadSteam3(depots: List): Unit = coroutineScope { - cdnClientPool?.updateServerList() + val maxNumServers = maxDownloads.coerceIn(20, 64) // Hard clamp at 64. Not sure how high we can go. + cdnClientPool?.updateServerList(maxNumServers) val downloadCounter = GlobalDownloadCounter() val depotsToDownload = ArrayList(depots.size) From 4005a69fd796803ba572c31f23e48e4f3e0f9ae8 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Fri, 14 Nov 2025 15:54:28 -0600 Subject: [PATCH 17/44] Add basic support for downloading Workshop collections --- .../depotdownloader/DepotDownloader.kt | 58 +++++++++++++++---- .../depotdownloader/Steam3Session.kt | 1 + 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index e98cd1c2..78b91640 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -13,6 +13,7 @@ import `in`.dragonbra.javasteam.depotdownloader.data.UgcItem import `in`.dragonbra.javasteam.enums.EAccountType import `in`.dragonbra.javasteam.enums.EAppInfoSection import `in`.dragonbra.javasteam.enums.EDepotFileFlag +import `in`.dragonbra.javasteam.enums.EWorkshopFileType import `in`.dragonbra.javasteam.steam.cdn.ClientLancache import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.steamapps.License @@ -118,6 +119,15 @@ class DepotDownloader @JvmOverloads constructor( const val DEFAULT_DOWNLOAD_DIR: String = "depots" val STAGING_DIR: Path = CONFIG_DIR.toPath() / "staging" + + private val SupportedWorkshopFileTypes: Set = setOf( + EWorkshopFileType.Community, + EWorkshopFileType.Art, + EWorkshopFileType.Screenshot, + EWorkshopFileType.Merch, + EWorkshopFileType.IntegratedGuide, + EWorkshopFileType.ControllerBinding, + ) } private val completionFuture = CompletableFuture() @@ -189,27 +199,55 @@ class DepotDownloader @JvmOverloads constructor( // region [REGION] Downloading Operations + private suspend fun processPublishedFile( + appId: Int, + publishedFileId: Long, + fileUrls: MutableList>, + contentFileIds: MutableList, + ) { + val details = steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) + val fileType = EWorkshopFileType.from(details!!.fileType) + + if (fileType == EWorkshopFileType.Collection) { + details.childrenList.forEach { child -> + processPublishedFile(appId, child.publishedfileid, fileUrls, contentFileIds) + } + } else if (SupportedWorkshopFileTypes.contains(fileType)) { + if (details.fileUrl.isNotEmpty()) { + fileUrls.add(Pair(details.filename, details.fileUrl)) + } else if (details.hcontentFile > 0) { + contentFileIds.add(details.hcontentFile) + } else { + logger?.error("Unable to locate manifest ID for published file $publishedFileId") + } + } else { + logger?.error("Published file $publishedFileId has unsupported file type $fileType. Skipping file") + } + } + @Throws(IllegalStateException::class) private suspend fun downloadPubFile(appId: Int, publishedFileId: Long) { - val details = requireNotNull( - steam3!!.getPublishedFileDetails(appId, PublishedFileID(publishedFileId)) - ) { "Pub File Null" } + val fileUrls = mutableListOf>() + val contentFileIds = mutableListOf() + + processPublishedFile(appId, publishedFileId, fileUrls, contentFileIds) - if (!details.fileUrl.isNullOrBlank()) { - downloadWebFile(appId, details.filename, details.fileUrl) - } else if (details.hcontentFile > 0) { + fileUrls.forEach { item -> + downloadWebFile(appId, item.first, item.second) + } + + if (contentFileIds.isNotEmpty()) { + val depotManifestIds = contentFileIds.map { id -> appId to id } downloadApp( appId = appId, - depotManifestIds = listOf(appId to details.hcontentFile), + depotManifestIds = depotManifestIds, branch = DEFAULT_BRANCH, os = null, arch = null, language = null, lv = false, - isUgc = true, + isUgc = true ) - } else { - logger?.error("Unable to locate manifest ID for published file $publishedFileId") } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index 1afcf241..c78ac43b 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -319,6 +319,7 @@ class Steam3Session( val pubFileRequest = SteammessagesPublishedfileSteamclient.CPublishedFile_GetDetails_Request.newBuilder().apply { this.appid = appId + this.includechildren = true this.addPublishedfileids(pubFile.toLong()) }.build() From 89db275c3be195f25e022d653d96a01380025812 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Tue, 18 Nov 2025 13:19:20 -0600 Subject: [PATCH 18/44] Add some more logs --- .../dragonbra/javasteam/depotdownloader/CDNClientPool.kt | 4 ++++ .../javasteam/depotdownloader/DepotConfigStore.kt | 9 +++++++++ .../javasteam/depotdownloader/DepotDownloader.kt | 9 +++++---- .../dragonbra/javasteam/depotdownloader/Steam3Session.kt | 8 ++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt index bd24d98d..ce7ec2d5 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/CDNClientPool.kt @@ -55,6 +55,8 @@ class CDNClientPool( } override fun close() { + logger?.debug("Closing...") + servers.set(emptyList()) cdnClient = null @@ -107,6 +109,7 @@ class CDNClientPool( fun returnConnection(server: Server?) { if (server == null) { + logger?.error("null server returned to cdn pool.") return } @@ -117,6 +120,7 @@ class CDNClientPool( fun returnBrokenConnection(server: Server?) { if (server == null) { + logger?.error("null broken server returned to pool") return } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt index 0bfebe14..327692d9 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotConfigStore.kt @@ -1,9 +1,12 @@ package `in`.dragonbra.javasteam.depotdownloader import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException import kotlinx.serialization.json.Json import okio.FileSystem +import okio.IOException import okio.Path +import kotlin.IllegalArgumentException /** * Singleton storage for tracking installed depot manifests. @@ -27,6 +30,11 @@ data class DepotConfigStore( private val json = Json { prettyPrint = true } + @Throws( + IOException::class, + SerializationException::class, + IllegalArgumentException::class, + ) fun loadFromFile(path: Path) { instance = if (FileSystem.SYSTEM.exists(path)) { FileSystem.SYSTEM.read(path) { @@ -39,6 +47,7 @@ data class DepotConfigStore( filePath = path } + @Throws(IllegalArgumentException::class) fun save() { val currentInstance = requireNotNull(instance) { "Saved config before loading" } val currentPath = requireNotNull(filePath) { "File path not set" } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 78b91640..f507458d 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -288,7 +288,7 @@ class DepotDownloader @JvmOverloads constructor( val (success, installDir) = createDirectories(appId, 0, appId) if (!success) { - logger?.debug("Error: Unable to create install directories!") + logger?.error("Error: Unable to create install directories!") return } @@ -337,12 +337,13 @@ class DepotDownloader @JvmOverloads constructor( filesystem.atomicMove(fileStagingPath, fileFinalPath) logger?.debug("File '$fileStagingPath' moved to final location: $fileFinalPath") } catch (e: IOException) { + logger?.error(e) throw e } } // L4D2 (app) supports LV - @Throws(IllegalStateException::class) + @Throws(IllegalStateException::class, DepotDownloaderException::class) private suspend fun downloadApp( appId: Int, depotManifestIds: List>, @@ -674,7 +675,7 @@ class DepotDownloader @JvmOverloads constructor( return depotChild["depotfromapp"].asInteger() } - @Throws(IllegalStateException::class) + @Throws(IllegalStateException::class, IOException::class) private fun createDirectories(depotId: Int, depotVersion: Int, appId: Int = 0): DirectoryResult { var installDir: Path? try { @@ -1593,7 +1594,7 @@ class DepotDownloader @JvmOverloads constructor( } if (ClientLancache.useLanCacheServer) { - logger?.debug("Detected Lan-Cache server! Downloads will be directed through the Lancache.") + logger?.debug("Detected Lan-Cache server! Downloads will be directed through the LanCache.") if (maxDownloads == 8) { maxDownloads = 25 } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt index c78ac43b..de86be16 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/Steam3Session.kt @@ -107,6 +107,7 @@ class Steam3Session( suspend fun requestAppInfo(appId: Int, bForce: Boolean = false) { if ((appInfo.containsKey(appId) && !bForce) || isAborted) { + logger?.debug("requestAppInfo already has $appId or is aborting") return } @@ -204,6 +205,7 @@ class Steam3Session( suspend fun requestDepotKey(depotId: Int, appId: Int = 0) { if (depotKeys.containsKey(depotId) || isAborted) { + logger?.debug("requestDepotKey already has $depotId or is aborting.") return } @@ -215,6 +217,7 @@ class Steam3Session( ) if (depotKey.result != EResult.OK) { + logger?.error("requestDepotKey result was ${depotKey.result}") return } @@ -228,6 +231,7 @@ class Steam3Session( branch: String, ): ULong = withContext(Dispatchers.IO) { if (isAborted) { + logger?.debug("getDepotManifestRequestCode aborting.") return@withContext 0UL } @@ -262,12 +266,14 @@ class Steam3Session( val cdnKey = depotId to server.host!! if (cdnAuthTokens.containsKey(cdnKey)) { + logger?.debug("requestCDNAuthToken already has $cdnKey") return@withContext } val completion = CompletableDeferred() if (isAborted || cdnAuthTokens.putIfAbsent(cdnKey, completion) != null) { + logger?.debug("requestCDNAuthToken is aborting or unable to map $cdnKey") return@withContext } @@ -278,6 +284,7 @@ class Steam3Session( logger?.debug("Got CDN auth token for ${server.host} result: ${cdnAuth.result} (expires ${cdnAuth.expiration})") if (cdnAuth.result != EResult.OK) { + logger?.error("requestCDNAuthToken result was ${cdnAuth.result}") return@withContext } @@ -342,6 +349,7 @@ class Steam3Session( if (callback.result == EResult.OK) { return callback } else if (callback.result == EResult.FileNotFound) { + logger?.error("getUGCDetails got FileNotFound for ${ugcHandle.value}") return null } From 35da3992581ed0b9dbeb3c5365170bc14c21f114 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Tue, 18 Nov 2025 15:39:45 -0600 Subject: [PATCH 19/44] Actually populate packageTokens. --- .../javasteam/depotdownloader/DepotDownloader.kt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index f507458d..20428057 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -187,10 +187,15 @@ class DepotDownloader @JvmOverloads constructor( logger = LogManager.getLogger(DepotDownloader::class.java) } - logger?.debug("DepotDownloader launched with ${licenses.size} for account") - steam3 = Steam3Session(steamClient, debug) + logger?.debug("DepotDownloader launched with ${licenses.size} for account") + licenses.forEach { license -> + if (license.accessToken.toULong() > 0UL) { + steam3!!.packageTokens[license.packageID] = license.accessToken + } + } + // Launch the processing loop scope.launch { processItems() From 415e584a01b4efaca6d330655e95fb7a4a52eb21 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 19 Nov 2025 00:28:53 -0600 Subject: [PATCH 20/44] use okhttp timeouts, optimize a performance chokepoint in manifest serialization, and update sample. --- .../_023_downloadapp/SampleDownloadApp.java | 18 ++++- .../dragonbra/javasteam/steam/cdn/Client.kt | 74 ++++++++----------- .../javasteam/types/DepotManifest.kt | 34 ++++----- 3 files changed, 63 insertions(+), 63 deletions(-) diff --git a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java index b6ec70b4..1305d187 100644 --- a/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java +++ b/javasteam-samples/src/main/java/in/dragonbra/javasteamsamples/_023_downloadapp/SampleDownloadApp.java @@ -18,8 +18,10 @@ import in.dragonbra.javasteam.steam.steamclient.callbackmgr.CallbackManager; import in.dragonbra.javasteam.steam.steamclient.callbacks.ConnectedCallback; import in.dragonbra.javasteam.steam.steamclient.callbacks.DisconnectedCallback; +import in.dragonbra.javasteam.steam.steamclient.configuration.SteamConfiguration; import in.dragonbra.javasteam.util.log.DefaultLogListener; import in.dragonbra.javasteam.util.log.LogManager; +import okhttp3.OkHttpClient; import org.jetbrains.annotations.NotNull; import java.io.Closeable; @@ -29,6 +31,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.CancellationException; +import java.util.concurrent.TimeUnit; /** * @author Oxters @@ -84,7 +87,18 @@ public void run() { // Most everything has been described in earlier samples. // Anything pertaining to this sample will be commented. - steamClient = new SteamClient(); + // Depot chunks are downloaded using OKHttp, it's best to set some timeouts. + var config = SteamConfiguration.create(builder -> { + builder.withHttpClient( + new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) // Time to establish connection + .readTimeout(60, TimeUnit.SECONDS) // Max inactivity between reads + .writeTimeout(30, TimeUnit.SECONDS) // Time for writes + .build() + ); + }); + + steamClient = new SteamClient(config); manager = new CallbackManager(steamClient); @@ -238,7 +252,7 @@ private void onLoggedOff(LoggedOffCallback callback) { private void downloadApp() { // Initiate the DepotDownloader, it is a Closable so it can be cleaned up when no longer used. // You will need to subscribe to LicenseListCallback to obtain your app licenses. - try (var depotDownloader = new DepotDownloader(steamClient, licenseList, false)) { + try (var depotDownloader = new DepotDownloader(steamClient, licenseList, true)) { // Add this class as a listener of IDownloadListener depotDownloader.addListener(this); diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index f77917e4..c34da41c 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -14,7 +14,6 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -import kotlinx.coroutines.withTimeout import okhttp3.HttpUrl import okhttp3.OkHttpClient import okhttp3.Request @@ -39,16 +38,6 @@ class Client(steamClient: SteamClient) : Closeable { private val logger: Logger = LogManager.getLogger(Client::class.java) - /** - * Default timeout to use when making requests - */ - var requestTimeout = 10_000L - - /** - * Default timeout to use when reading the response body - */ - var responseBodyTimeout = 60_000L - private fun buildCommand( server: Server, command: String, @@ -144,9 +133,7 @@ class Client(steamClient: SteamClient) : Closeable { logger.debug("Request URL is: $request") try { - val response = withTimeout(requestTimeout) { - httpClient.newCall(request).executeAsync() - } + val response = httpClient.newCall(request).executeAsync() if (!response.isSuccessful) { throw SteamKitWebRequestException( @@ -155,30 +142,28 @@ class Client(steamClient: SteamClient) : Closeable { ) } - return@withContext withTimeout(responseBodyTimeout) { - response.use { resp -> - val responseBody = resp.body?.bytes() - ?: throw SteamKitWebRequestException("Response body is null") + return@withContext response.use { resp -> + val responseBody = resp.body?.bytes() + ?: throw SteamKitWebRequestException("Response body is null") - if (responseBody.isEmpty()) { - throw SteamKitWebRequestException("Response is empty") - } - - // Decompress the zipped manifest data - ZipInputStream(ByteArrayInputStream(responseBody)).use { zipInputStream -> - zipInputStream.nextEntry - ?: throw SteamKitWebRequestException("Expected the zip to contain at least one file") + if (responseBody.isEmpty()) { + throw SteamKitWebRequestException("Response is empty") + } - val manifestData = zipInputStream.readBytes() + // Decompress the zipped manifest data + ZipInputStream(ByteArrayInputStream(responseBody)).use { zipInputStream -> + zipInputStream.nextEntry + ?: throw SteamKitWebRequestException("Expected the zip to contain at least one file") - val depotManifest = DepotManifest.deserialize(ByteArrayInputStream(manifestData)) + val manifestData = zipInputStream.readBytes() - if (depotKey != null) { - depotManifest.decryptFilenames(depotKey) - } + val depotManifest = DepotManifest.deserialize(ByteArrayInputStream(manifestData)) - depotManifest + if (depotKey != null) { + depotManifest.decryptFilenames(depotKey) } + + depotManifest } } } catch (e: Exception) { @@ -240,9 +225,7 @@ class Client(steamClient: SteamClient) : Closeable { } try { - val response = withTimeout(requestTimeout) { - httpClient.newCall(request).executeAsync() - } + val response = httpClient.newCall(request).executeAsync() response.use { resp -> if (!resp.isSuccessful) { @@ -252,16 +235,23 @@ class Client(steamClient: SteamClient) : Closeable { ) } - val contentLength = resp.body.contentLength().toInt() + var contentLength = chunk.compressedLength - if (contentLength == 0) { - chunk.compressedLength - } + if (resp.body.contentLength().toInt() > 0) { + contentLength = resp.body.contentLength().toInt() - // Validate content length - if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { + // assert that lengths match only if the chunk has a length assigned. + if (chunk.compressedLength > 0 && contentLength != chunk.compressedLength) { + throw SteamKitWebRequestException( + "Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})" + ) + } + } else if (contentLength > 0) { + logger.debug("Response does not have Content-Length, falling back to chunk.CompressedLength.") + } else { throw SteamKitWebRequestException( - "Content-Length mismatch for depot chunk! (was $contentLength, but should be ${chunk.compressedLength})" + "Response does not have Content-Length and chunk.CompressedLength is not set.", + response ) } diff --git a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt index b747d0c6..10233687 100644 --- a/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt +++ b/src/main/java/in/dragonbra/javasteam/types/DepotManifest.kt @@ -24,7 +24,6 @@ import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec import kotlin.NoSuchElementException import kotlin.collections.ArrayList -import kotlin.collections.HashSet /** * Represents a Steam3 depot manifest. @@ -392,25 +391,23 @@ class DepotManifest { * @param output The stream to which the serialized depot manifest will be written. */ fun serialize(output: OutputStream) { - val payload = ContentManifestPayload.newBuilder() - val uniqueChunks = object : HashSet() { - // This acts like "ChunkIdComparer" - private val items = mutableListOf() - - override fun add(element: ByteArray): Boolean { - if (contains(element)) return false - items.add(element) - return true + // Basically like ChunkIdComparer from DepotDownloader + class ByteArrayKey(private val byteArray: ByteArray) { + override fun equals(other: Any?): Boolean = other is ByteArrayKey && byteArray.contentEquals(other.byteArray) + + override fun hashCode(): Int = if (byteArray.size >= 4) { + ((byteArray[0].toInt() and 0xFF)) or + ((byteArray[1].toInt() and 0xFF) shl 8) or + ((byteArray[2].toInt() and 0xFF) shl 16) or + ((byteArray[3].toInt() and 0xFF) shl 24) + } else { + byteArray.contentHashCode() } - - override fun contains(element: ByteArray): Boolean = items.any { it.contentEquals(element) } - - override fun iterator(): MutableIterator = items.iterator() - - override val size: Int - get() = items.size } + val payload = ContentManifestPayload.newBuilder() + val uniqueChunks = hashSetOf() + // Reuse instance. val sha1Digest = MessageDigest.getInstance("SHA-1", CryptoHelper.SEC_PROV) @@ -428,7 +425,6 @@ class DepotManifest { protofile.filename = file.fileName.replace('/', '\\') protofile.shaFilename = ByteString.copyFrom( CryptoHelper.shaHash( - sha1Digest, file.fileName .replace('/', '\\') .lowercase(Locale.getDefault()) @@ -453,7 +449,7 @@ class DepotManifest { }.build() protofile.addChunks(protochunk) - uniqueChunks.add(chunk.chunkID!!) + uniqueChunks.add(ByteArrayKey(chunk.chunkID!!)) } payload.addMappings(protofile.build()) From 5f96c355ca6fd46c0836d87c11c1dc2fa02b178f Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Thu, 20 Nov 2025 12:20:20 -0600 Subject: [PATCH 21/44] Temp work around. Don't cancel on a depot that has no manifest request code. This may have unforseen issues, but stops some games from failing because depot 1523211 doesnt have a code. --- .../dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 20428057..1bb4ccc3 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -928,7 +928,10 @@ class DepotDownloader @JvmOverloads constructor( // If we could not get the manifest code, this is a fatal error if (manifestRequestCode == 0UL) { - cancel("manifestRequestCode is 0UL") + // TODO this should be a fatal error and bail out. But I guess we can continue. + logger?.error("Manifest request code is 0. Skipping depot ${depot.depotId}") + return@withContext null + // cancel("manifestRequestCode is 0UL") } } From c52354c12322a490255ab56d1d7c38bbb61f4eb5 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 30 Nov 2025 21:42:32 -0600 Subject: [PATCH 22/44] Clarify some error logs. --- .../javasteam/depotdownloader/DepotDownloader.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 1bb4ccc3..b45d0b32 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -342,7 +342,7 @@ class DepotDownloader @JvmOverloads constructor( filesystem.atomicMove(fileStagingPath, fileFinalPath) logger?.debug("File '$fileStagingPath' moved to final location: $fileFinalPath") } catch (e: IOException) { - logger?.error(e) + logger?.error("Failed to move files", e) throw e } } @@ -734,7 +734,7 @@ class DepotDownloader @JvmOverloads constructor( filesystem.createDirectories(installDir / STAGING_DIR) } } catch (e: IOException) { - logger?.error(e) + logger?.error("Failed to create directory for depot $depotId, app $appId", e) return DirectoryResult(false, null) } @@ -950,7 +950,7 @@ class DepotDownloader @JvmOverloads constructor( cdnClientPool!!.returnConnection(connection) } catch (e: CancellationException) { // logger?.error("Connection timeout downloading depot manifest ${depot.depotId} ${depot.manifestId}. Retrying.") - logger?.error(e) + logger?.error("Cancellation Exception thrown in process manifest", e) break } catch (e: SteamKitWebRequestException) { // If the CDN returned 403, attempt to get a cdn auth if we didn't yet @@ -1414,7 +1414,7 @@ class DepotDownloader @JvmOverloads constructor( break } catch (e: CancellationException) { - logger?.error(e) + logger?.error("Cancellation exception in download depot file chunk", e) } catch (e: SteamKitWebRequestException) { // If the CDN returned 403, attempt to get a cdn auth if we didn't yet, // if auth task already exists, make sure it didn't complete yet, so that it gets awaited above @@ -1560,7 +1560,7 @@ class DepotDownloader @JvmOverloads constructor( processingChannel.send(item) notifyListeners { it.onItemAdded(item) } } catch (e: Exception) { - logger?.error(e) + logger?.error("Could not add item ${item.appId}", e) throw e } } @@ -1577,7 +1577,7 @@ class DepotDownloader @JvmOverloads constructor( notifyListeners { it.onItemAdded(item) } } } catch (e: Exception) { - logger?.error(e) + logger?.error("Could not add all files", e) throw e } } From 0af6df6768037841fec118bf38a6689bf0ea0c16 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Sun, 30 Nov 2025 22:06:31 -0600 Subject: [PATCH 23/44] Properly install games based on install directory name. --- .../depotdownloader/DepotDownloader.kt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index b45d0b32..cbe46037 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -694,13 +694,14 @@ class DepotDownloader @JvmOverloads constructor( filesystem.createDirectories(DEFAULT_DOWNLOAD_DIR.toPath()) if (config.installToGameNameDirectory) { - val gameName = getAppName(appId) + val info = getSteam3AppSection(appId, EAppInfoSection.Config) ?: KeyValue.INVALID + val appInstallDir = info["installdir"].asString() - if (gameName.isBlank()) { - throw IOException("Game name is blank, cannot create directory") + if (appInstallDir.isNullOrBlank()) { + throw IOException("Config install directory is blank, cannot create directory") } - installDir = DEFAULT_DOWNLOAD_DIR.toPath() / gameName + installDir = DEFAULT_DOWNLOAD_DIR.toPath() / appInstallDir filesystem.createDirectories(installDir) } else { @@ -717,13 +718,14 @@ class DepotDownloader @JvmOverloads constructor( filesystem.createDirectories(config.installPath!!) if (config.installToGameNameDirectory) { - val gameName = getAppName(appId) + val info = getSteam3AppSection(appId, EAppInfoSection.Config) ?: KeyValue.INVALID + val appInstallDir = info["installdir"].asString() - if (gameName.isBlank()) { - throw IOException("Game name is blank, cannot create directory") + if (appInstallDir.isNullOrBlank()) { + throw IOException("Config install directory is blank, cannot create directory") } - installDir = config.installPath!! / gameName + installDir = config.installPath!! / appInstallDir filesystem.createDirectories(installDir) } else { From a101526e19b3d787c154e7786ccbc5fbb859d179 Mon Sep 17 00:00:00 2001 From: Utkarsh Dalal Date: Tue, 2 Dec 2025 15:51:28 +0530 Subject: [PATCH 24/44] Added onChunkCompleted callback --- .../depotdownloader/DepotDownloader.kt | 21 ++++++++++++++++++- .../depotdownloader/IDownloadListener.kt | 11 ++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index cbe46037..c9492a8d 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -1499,17 +1499,36 @@ class DepotDownloader @JvmOverloads constructor( logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) } else { - // Just update counters without notifying + // Update counters and notify on chunk completion + val sizeDownloaded: Long + val depotPercentage: Float + val compressedBytes: Long + val uncompressedBytes: Long + synchronized(depotDownloadCounter) { depotDownloadCounter.sizeDownloaded += written.toLong() depotDownloadCounter.depotBytesCompressed += chunk.compressedLength depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + + sizeDownloaded = depotDownloadCounter.sizeDownloaded + compressedBytes = depotDownloadCounter.depotBytesCompressed + uncompressedBytes = depotDownloadCounter.depotBytesUncompressed + depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) } synchronized(downloadCounter) { downloadCounter.totalBytesCompressed += chunk.compressedLength downloadCounter.totalBytesUncompressed += chunk.uncompressedLength } + + notifyListeners { listener -> + listener.onChunkCompleted( + depotId = depot.depotId, + depotPercentComplete = depotPercentage, + compressedBytes = compressedBytes, + uncompressedBytes = uncompressedBytes + ) + } } } diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt index 331f4a72..f005c6cb 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/IDownloadListener.kt @@ -48,6 +48,17 @@ interface IDownloadListener { */ fun onFileCompleted(depotId: Int, fileName: String, depotPercentComplete: Float) {} + /** + * Called when a chunk completes downloading. + * Provides more frequent progress updates than onFileCompleted. + * + * @param depotId The depot being downloaded + * @param depotPercentComplete Overall depot completion percentage (0f to 1f) + * @param compressedBytes Total compressed bytes downloaded so far for this depot + * @param uncompressedBytes Total uncompressed bytes downloaded so far for this depot + */ + fun onChunkCompleted(depotId: Int, depotPercentComplete: Float, compressedBytes: Long, uncompressedBytes: Long) {} + /** * Called when a depot finishes downloading. * Use this for printing summary like "Depot 228990 - Downloaded X bytes (Y bytes uncompressed)" From bfd5659a37bc1d7c9896322825982e46e2185562 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 17 Dec 2025 00:42:32 -0600 Subject: [PATCH 25/44] Add Client Communication for remote control. Add some methods to SteamContent. Update ProtoParser to support webui protobufs. All in hopes to being p2p downloads to depot downloader. --- .../generators/rpc/parser/ProtoParser.kt | 15 +- .../steamclientcommunication/ClientAppList.kt | 30 +++ .../ClientAppListAppData.kt | 90 +++++++ .../ClientAppListDlcData.kt | 21 ++ .../steamclientcommunication/ClientInfo.kt | 42 +++ .../ClientLogonInfo.kt | 18 ++ .../ClientLogonInfoSession.kt | 37 +++ .../InstalledAppsFilter.kt | 13 + .../steamclientcommunication/RunningGames.kt | 21 ++ .../SteamClientCommunication.kt | 253 ++++++++++++++++++ .../steamcontent/ClientUpdateHosts.kt | 13 + .../handlers/steamcontent/DepotPatchInfo.kt | 13 + .../steamcontent/GetPeerContentInfo.kt | 11 + .../steamcontent/RequestPeerContentServer.kt | 13 + .../handlers/steamcontent/SteamContent.kt | 119 +++++++- .../protobufs/webui/service_clientcomm.proto | 175 ++++++++++++ .../javasteam/rpc/UnifiedInterfaceTest.kt | 3 + 17 files changed, 880 insertions(+), 7 deletions(-) create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt create mode 100644 src/main/proto/in/dragonbra/javasteam/protobufs/webui/service_clientcomm.proto diff --git a/buildSrc/src/main/kotlin/in/dragonbra/generators/rpc/parser/ProtoParser.kt b/buildSrc/src/main/kotlin/in/dragonbra/generators/rpc/parser/ProtoParser.kt index 0d7ed77c..af0264ba 100644 --- a/buildSrc/src/main/kotlin/in/dragonbra/generators/rpc/parser/ProtoParser.kt +++ b/buildSrc/src/main/kotlin/in/dragonbra/generators/rpc/parser/ProtoParser.kt @@ -14,6 +14,7 @@ class ProtoParser(private val outputDir: File) { private val suppressAnnotation = AnnotationSpec .builder(Suppress::class) .addMember("%S", "KDocUnresolvedReference") // IntelliJ's seems to get confused with canonical names + .addMember("%S", "RemoveRedundantQualifierName") // Full Qualifier names are fine .addMember("%S", "RedundantVisibilityModifier") // KotlinPoet is an explicit API generator .addMember("%S", "unused") // All methods could be used. .build() @@ -85,6 +86,8 @@ class ProtoParser(private val outputDir: File) { private fun buildClass(file: File, service: Service) { val protoFileName = transformProtoFileName(file.name) + val parentPathName = file.parentFile.name + // Class Builder val steamUnifiedMessagesClassName = ClassName( "in.dragonbra.javasteam.steam.handlers.steamunifiedmessages", @@ -129,7 +132,7 @@ class ProtoParser(private val outputDir: File) { // HAS Response numResponse++ val className = ClassName( - packageName = "in.dragonbra.javasteam.protobufs.steamclient.$protoFileName", + packageName = "in.dragonbra.javasteam.protobufs.$parentPathName.$protoFileName", method.responseType ) responseBlock.addStatement( @@ -141,7 +144,7 @@ class ProtoParser(private val outputDir: File) { // NO Response numNotification++ val className = ClassName( - packageName = "in.dragonbra.javasteam.protobufs.steamclient.$protoFileName", + packageName = "in.dragonbra.javasteam.protobufs.$parentPathName.$protoFileName", method.requestType ) notificationBlock.addStatement( @@ -192,7 +195,7 @@ class ProtoParser(private val outputDir: File) { .addParameter( "request", ClassName( - packageName = "in.dragonbra.javasteam.protobufs.steamclient.$protoFileName", + packageName = "in.dragonbra.javasteam.protobufs.$parentPathName.$protoFileName", method.requestType ) ) @@ -207,14 +210,14 @@ class ProtoParser(private val outputDir: File) { packageName = "in.dragonbra.javasteam.steam.handlers.steamunifiedmessages.callback", "ServiceMethodResponse" ).parameterizedBy( - ClassName.bestGuess("in.dragonbra.javasteam.protobufs.steamclient.$protoFileName.${method.responseType}.Builder") + ClassName.bestGuess("in.dragonbra.javasteam.protobufs.$parentPathName.$protoFileName.${method.responseType}.Builder") ) ) ) funcBuilder.addStatement( format = "return unifiedMessages!!.sendMessage(\n%T.Builder::class.java,\n%S,\nrequest\n)", ClassName( - packageName = "in.dragonbra.javasteam.protobufs.steamclient.$protoFileName", + packageName = "in.dragonbra.javasteam.protobufs.$parentPathName.$protoFileName", method.responseType ), "${service.name}.${method.methodName}#1" @@ -223,7 +226,7 @@ class ProtoParser(private val outputDir: File) { funcBuilder.addStatement( format = "unifiedMessages!!.sendNotification<%T.Builder>(\n%S,\nrequest\n)", ClassName( - packageName = "in.dragonbra.javasteam.protobufs.steamclient.$protoFileName", + packageName = "in.dragonbra.javasteam.protobufs.$parentPathName.$protoFileName", method.requestType ), "${service.name}.${method.methodName}#1" diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt new file mode 100644 index 00000000..2bbe8ecb --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt @@ -0,0 +1,30 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +/** + * TODO kdoc + * @param bytesAvailable + * @param apps + * @param clientInfo + * @param refetchIntervalSecFull + * @param refetchIntervalSecChanging + * @param refetchIntervalSecUpdating + */ +data class ClientAppList( + val bytesAvailable: Long, + val apps: List, + val clientInfo: ClientInfo, + val refetchIntervalSecFull: Int, + val refetchIntervalSecChanging: Int, + val refetchIntervalSecUpdating: Int, +) { + override fun toString(): String = """ + ClientAppList( + bytesAvailable=$bytesAvailable, + apps=$apps, + clientInfo=$clientInfo, + refetchIntervalSecFull=$refetchIntervalSecFull, + refetchIntervalSecChanging=$refetchIntervalSecChanging, + refetchIntervalSecUpdating=$refetchIntervalSecUpdating + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt new file mode 100644 index 00000000..eb2ea9bd --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt @@ -0,0 +1,90 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +/** + * TODO kdoc + * @param appid + * @param app + * @param category + * @param appType + * @param numDownloading + * @param bytesDownloadRate + * @param bytesDownloaded + * @param bytesToDownload + * @param dlcs + * @param favorite + * @param autoUpdate + * @param installed + * @param downloadPaused + * @param changing + * @param availableOnPlatform + * @param bytesStaged + * @param bytesToStage + * @param bytesRequired + * @param sourceBuildId + * @param targetBuildId + * @param estimatedSecondsRemaining + * @param queuePosition + * @param uninstalling + * @param rtTimeScheduled + * @param running + * @param updatePercentage + */ +data class ClientAppListAppData( + val appid: Int, + val app: String, + val category: String, + val appType: String, + val numDownloading: Int, + val bytesDownloadRate: Int, + val bytesDownloaded: Long, + val bytesToDownload: Long, + val dlcs: List, + val favorite: Boolean, + val autoUpdate: Boolean, + val installed: Boolean, + val downloadPaused: Boolean, + val changing: Boolean, + val availableOnPlatform: Boolean, + val bytesStaged: Long, + val bytesToStage: Long, + val bytesRequired: Long, + val sourceBuildId: Int, + val targetBuildId: Int, + val estimatedSecondsRemaining: Int, + val queuePosition: Int, + val uninstalling: Boolean, + val rtTimeScheduled: Int, + val running: Boolean, + val updatePercentage: Int, +) { + override fun toString(): String = """ + ClientAppListAppData( + appid=$appid, + app='$app', c + ategory='$category', + appType='$appType', + numDownloading=$numDownloading, + bytesDownloadRate=$bytesDownloadRate, + bytesDownloaded=$bytesDownloaded, + bytesToDownload=$bytesToDownload, + dlcs=$dlcs, + favorite=$favorite, + autoUpdate=$autoUpdate, + installed=$installed, + downloadPaused=$downloadPaused, + changing=$changing, + availableOnPlatform=$availableOnPlatform, + bytesStaged=$bytesStaged, + bytesToStage=$bytesToStage, + bytesRequired=$bytesRequired, + sourceBuildId=$sourceBuildId, + targetBuildId=$targetBuildId, + estimatedSecondsRemaining=$estimatedSecondsRemaining, + queuePosition=$queuePosition, + uninstalling=$uninstalling, + rtTimeScheduled=$rtTimeScheduled, + running=$running, + updatePercentage=$updatePercentage + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt new file mode 100644 index 00000000..525246ea --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt @@ -0,0 +1,21 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +/** + * TODO kdoc + * @param appId + * @param app + * @param installed + */ +data class ClientAppListDlcData( + val appId: Int, + val app: String, + val installed: Int, +) { + override fun toString(): String = """ + ClientAppListDlcData( + appId=$appId, + app='$app', + installed=$installed + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt new file mode 100644 index 00000000..a7343f5e --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt @@ -0,0 +1,42 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +/** + * TODO kdoc + * @param packageVersion + * @param os + * @param machineName + * @param ipPublic + * @param ipPrivate + * @param bytesAvailable + * @param runningGames + * @param protocolVersion + * @param clientCommVersion + * @param localUsers + */ +data class ClientInfo( + val packageVersion: Int, + val os: String, + val machineName: String, + val ipPublic: String, + val ipPrivate: String, + val bytesAvailable: Long, + val runningGames: List, + val protocolVersion: Int, + val clientCommVersion: Int, + val localUsers: List, +) { + override fun toString(): String = """ + ClientInfo( + packageVersion=$packageVersion, + os='$os', + machineName='$machineName', + ipPublic='$ipPublic', + ipPrivate='$ipPrivate', + bytesAvailable=$bytesAvailable, + runningGames=$runningGames, + protocolVersion=$protocolVersion, + clientCommVersion=$clientCommVersion, + localUsers=$localUsers + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt new file mode 100644 index 00000000..16401061 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt @@ -0,0 +1,18 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +/** + * TODO kdoc + * @param sessions + * @param refetchIntervalSec + */ +data class ClientLogonInfo( + val sessions: List, + val refetchIntervalSec: Int, +) { + override fun toString(): String = """ + ClientLogonInfo( + sessions=$sessions, + refetchIntervalSec=$refetchIntervalSec + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt new file mode 100644 index 00000000..fac90b8d --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt @@ -0,0 +1,37 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +import `in`.dragonbra.javasteam.enums.EGamingDeviceType +import `in`.dragonbra.javasteam.enums.EOSType +import `in`.dragonbra.javasteam.enums.ESteamRealm + +/** + * Describes an active device running desktop Steam client. + * @param clientInstanceId Instance ID that is used to send remote signals + * @param protocolVersion Protocol version of the client + * @param osName Name of installed OS, such as "Linux 6.x" + * @param machineName Name of the device, such as "steamdeck" + * @param osType Type of installed OS, such as [EOSType].k_Linux6x + * @param deviceType Type of the device, such as [EGamingDeviceType].k_EGamingDeviceType_SteamDeck + * @param realm Realm of the session from. See [ESteamRealm]. + */ +data class ClientLogonInfoSession( + val clientInstanceId: Long, + val protocolVersion: Int, + val osName: String, + val machineName: String, + val osType: Int, + val deviceType: Int, + val realm: Int, +) { + override fun toString(): String = """ + ClientLogonInfoSession( + clientInstanceId=$clientInstanceId, + protocolVersion=$protocolVersion, + osName='$osName', + machineName='$machineName', + osType=$osType, + deviceType=$deviceType, + realm=$realm + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt new file mode 100644 index 00000000..5491c0fa --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt @@ -0,0 +1,13 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +enum class InstalledAppsFilter { + /** + * Return everything, including not installed apps + */ + None, + + /** + * Return only apps that are "in progress" - downloading, updated, scheduled + */ + Changing, +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt new file mode 100644 index 00000000..d04b97fd --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt @@ -0,0 +1,21 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +/** + * TODO kdoc + * @param appId + * @param extraInfo + * @param timeRunningSec + */ +data class RunningGames( + val appId: Int, + val extraInfo: String, + val timeRunningSec: Int, +) { + override fun toString(): String = """ + RunningGames( + appId=$appId, + extraInfo='$extraInfo', + timeRunningSec=$timeRunningSec + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt new file mode 100644 index 00000000..29732ad7 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt @@ -0,0 +1,253 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +import `in`.dragonbra.javasteam.base.IPacketMsg +import `in`.dragonbra.javasteam.enums.EResult +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_EnableOrDisableDownloads_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_GetAllClientLogonInfo_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_GetClientAppList_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_InstallClientApp_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_LaunchClientApp_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_SetClientAppUpdateState_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_UninstallClientApp_Request +import `in`.dragonbra.javasteam.rpc.service.ClientComm +import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler +import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.async + +/** + * Allows controlling of other running Steam clients. + */ +class SteamClientCommunication : ClientMsgHandler() { + + private val clientComm: ClientComm by lazy { + val unifiedMessages = client.getHandler() + ?: throw NullPointerException("Unable to get SteamUnifiedMessages handler") + unifiedMessages.createService() + } + + /** + * Return the list of active devices that are running any Steam client and connected to the network. + * Note: this will return all connected clients. Filter the results based on OS or device type. + */ + fun getAllClientLogonInfo(): Deferred = client.defaultScope.async { + val request = CClientComm_GetAllClientLogonInfo_Request.newBuilder().build() + + val message = clientComm.getAllClientLogonInfo(request).await() + val response = message.body.build() + + return@async ClientLogonInfo( + sessions = response.sessionsList.map { + ClientLogonInfoSession( + clientInstanceId = it.clientInstanceid, + protocolVersion = it.protocolVersion, + osName = it.osName, + machineName = it.machineName, + osType = it.osType, + deviceType = it.deviceType, + realm = it.realm, + ) + }, + refetchIntervalSec = response.refetchIntervalSec, + ) + } + + /** + * Return the list of applications of the remote device. + * This is not the list of downloaded apps, but the whole "available" list with a flag indicating about it being downloaded. + * @param remoteId remote session ID + * @param filters filters + * @param language language + */ + @JvmOverloads + fun getClientAppList( + remoteId: Long, + filters: InstalledAppsFilter = InstalledAppsFilter.None, + language: String = "english", + ): Deferred = client.defaultScope.async { + val request = CClientComm_GetClientAppList_Request.newBuilder().apply { + this.clientInstanceid = remoteId + this.language = language + this.includeClientInfo = true + this.fields = "games" + this.filters = when (filters) { + InstalledAppsFilter.None -> "none" + InstalledAppsFilter.Changing -> "changing" + } + }.build() + + val message = clientComm.getClientAppList(request).await() + val response = message.body.build() + + return@async ClientAppList( + bytesAvailable = response.bytesAvailable, + apps = response.appsList.map { app -> + ClientAppListAppData( + appid = app.appid, + app = app.app, + category = app.category, + appType = app.appType, + numDownloading = app.numDownloading, + bytesDownloadRate = app.bytesDownloadRate, + bytesDownloaded = app.bytesDownloaded, + bytesToDownload = app.bytesToDownload, + dlcs = app.dlcsList.map { dlc -> + ClientAppListDlcData( + appId = dlc.appid, + app = dlc.app, + installed = dlc.installed, + ) + }, + favorite = app.favorite, + autoUpdate = app.autoUpdate, + installed = app.installed, + downloadPaused = app.downloadPaused, + changing = app.changing, + availableOnPlatform = app.availableOnPlatform, + bytesStaged = app.bytesStaged, + bytesToStage = app.bytesToStage, + bytesRequired = app.bytesRequired, + sourceBuildId = app.sourceBuildid, + targetBuildId = app.targetBuildid, + estimatedSecondsRemaining = app.estimatedSecondsRemaining, + queuePosition = app.queuePosition, + uninstalling = app.uninstalling, + rtTimeScheduled = app.rtTimeScheduled, + running = app.running, + updatePercentage = app.updatePercentage, + ) + }, + clientInfo = ClientInfo( + packageVersion = response.clientInfo.packageVersion, + os = response.clientInfo.os, + machineName = response.clientInfo.machineName, + ipPublic = response.clientInfo.ipPublic, + ipPrivate = response.clientInfo.ipPrivate, + bytesAvailable = response.clientInfo.bytesAvailable, + runningGames = response.clientInfo.runningGamesList.map { game -> + RunningGames( + appId = game.appid, + extraInfo = game.extraInfo, + timeRunningSec = game.timeRunningSec, + ) + }, + protocolVersion = response.clientInfo.protocolVersion, + clientCommVersion = response.clientInfo.clientcommVersion, + localUsers = response.clientInfo.localUsersList, + ), + refetchIntervalSecFull = response.refetchIntervalSecFull, + refetchIntervalSecChanging = response.refetchIntervalSecChanging, + refetchIntervalSecUpdating = response.refetchIntervalSecUpdating, + ) + } + + /** + * Adds the application to the remote installation queue. + * @param remoteId + * @param appId + * @return **true** if successfully, otherwise false. + */ + fun installClientApp(remoteId: Long, appId: Int): Deferred = client.defaultScope.async { + val request = CClientComm_InstallClientApp_Request.newBuilder().apply { + this.clientInstanceid = remoteId + this.appid = appId + }.build() + + val message = clientComm.installClientApp(request).await() + // Request has empty response. + + return@async message.result == EResult.OK + } + + /** + * Sets the update state of an app in remote installation queue. + * Action set to true will move the requested app to the top of the queue. + * @param remoteId + * @param appId + * @param action + * @return **true** if successfully, otherwise false. + */ + fun setClientAppUpdateState(remoteId: Long, appId: Int, action: Boolean): Deferred = + client.defaultScope.async { + val request = CClientComm_SetClientAppUpdateState_Request.newBuilder().apply { + this.clientInstanceid = remoteId + this.appid = appId + this.action = if (action) 1 else 0 + }.build() + + val message = clientComm.setClientAppUpdateState(request).await() + // Request has empty response. + + return@async message.result == EResult.OK + } + + /** + * Requests to uninstall the app from the device. + * @param remoteId + * @param appId + * @return **true** if successfully, otherwise false. + */ + fun uninstallClientApp(remoteId: Long, appId: Int): Deferred = client.defaultScope.async { + val request = CClientComm_UninstallClientApp_Request.newBuilder().apply { + this.clientInstanceid = remoteId + this.appid = appId + }.build() + + val message = clientComm.uninstallClientApp(request).await() + // Request has empty response. + + return@async message.result == EResult.OK + } + + /** + * Pauses or resumes active download - the first item in the queue. + * @param remoteId + * @param enable + * @return **true** if successfully, otherwise false. + */ + fun enableOrDisableDownloads(remoteId: Long, enable: Boolean): Deferred = client.defaultScope.async { + val request = CClientComm_EnableOrDisableDownloads_Request.newBuilder().apply { + this.clientInstanceid = remoteId + this.enable = enable + }.build() + + val message = clientComm.enableOrDisableDownloads(request).await() + // Request has empty response. + + return@async message.result == EResult.OK + } + + /** + * Launches the application on the remote device. + * @param remoteId + * @param appId + * @param parameters + * @return **true** if successfully, otherwise false. + */ + fun launchClientApp(remoteId: Long, appId: Int, parameters: String? = null): Deferred = + client.defaultScope.async { + val request = CClientComm_LaunchClientApp_Request.newBuilder().apply { + this.clientInstanceid = remoteId + this.appid = appId + this.queryParams = parameters.orEmpty() + }.build() + + val message = clientComm.launchClientApp(request).await() + // Request has empty response. + + return@async message.result == EResult.OK + } + + // TODO: GetClientLogonInfo() + + // TODO: GetClientInfo() + + /** + * Handles a client message. This should not be called directly. + * + * @param packetMsg The packet message that contains the data. + */ + override fun handleMsg(packetMsg: IPacketMsg) { + // not used + } +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt new file mode 100644 index 00000000..364dcee2 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt @@ -0,0 +1,13 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamcontent + +/** + * TODO kdoc + * @param hostsKv + * @param validUntilTime + * @param ipCountry + */ +data class ClientUpdateHosts( + val hostsKv: String, + val validUntilTime: Long, + val ipCountry: String, +) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt new file mode 100644 index 00000000..1ffb444e --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt @@ -0,0 +1,13 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamcontent + +/** + * TODO kdoc + * @param isAvailable + * @param patchSize + * @param patchedChunksSize + */ +data class DepotPatchInfo( + val isAvailable: Boolean, + val patchSize: Long, + val patchedChunksSize: Long, +) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt new file mode 100644 index 00000000..a6e125a7 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt @@ -0,0 +1,11 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamcontent + +/** + * TODO kdoc + * @param appIds + * @param ipPublic + */ +data class GetPeerContentInfo( + val appIds: List, + val ipPublic: String, +) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt new file mode 100644 index 00000000..60644c32 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt @@ -0,0 +1,13 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamcontent + +/** + * TODO kdoc + * @param serverPort + * @param installedDepots + * @param accessToken + */ +data class RequestPeerContentServer( + val serverPort: Int, + val installedDepots: List, + val accessToken: Long, +) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt index 5e66c4e6..073898f0 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt @@ -2,12 +2,15 @@ package `in`.dragonbra.javasteam.steam.handlers.steamcontent import `in`.dragonbra.javasteam.base.IPacketMsg import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetCDNAuthToken_Request +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetClientUpdateHosts_Request +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetDepotPatchInfo_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetManifestRequestCode_Request +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetPeerContentInfo_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetServersForSteamPipe_Request +import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_RequestPeerContentServer_Request import `in`.dragonbra.javasteam.rpc.service.ContentServerDirectory import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler -import `in`.dragonbra.javasteam.steam.handlers.steamcontent.CDNAuthToken import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages import `in`.dragonbra.javasteam.steam.webapi.ContentServerDirectoryService import kotlinx.coroutines.CoroutineScope @@ -48,6 +51,57 @@ class SteamContent : ClientMsgHandler() { return@async ContentServerDirectoryService.convertServerList(response) } + /** + * TODO kdoc + * @param appId + * @param depotId + * @param sourceManifestId + * @param targetManifestId + * @return A [DepotPatchInfo] + */ + fun getDepotPatchInfo( + appId: Int, + depotId: Int, + sourceManifestId: Long, + targetManifestId: Long, + parentScope: CoroutineScope, + ): Deferred = parentScope.async { + val request = CContentServerDirectory_GetDepotPatchInfo_Request.newBuilder().apply { + this.appid = appId + this.depotid = depotId + this.sourceManifestid = sourceManifestId + this.targetManifestid = targetManifestId + }.build() + + val message = contentService.getDepotPatchInfo(request).await() + val response = message.body.build() + + return@async DepotPatchInfo( + isAvailable = response.isAvailable, + patchSize = response.patchSize, + patchedChunksSize = response.patchedChunksSize + ) + } + + /** + * TODO kdoc + * @param cachedSignature + * @return A [ClientUpdateHosts] + */ + fun getClientUpdateHosts( + cachedSignature: String, + parentScope: CoroutineScope, + ): Deferred = parentScope.async { + val request = CContentServerDirectory_GetClientUpdateHosts_Request.newBuilder().apply { + this.cachedSignature = cachedSignature + }.build() + + val message = contentService.getClientUpdateHosts(request).await() + val response = message.body.build() + + return@async ClientUpdateHosts(response.hostsKv, response.validUntilTime, response.ipCountry) + } + /** * Request the manifest request code for the specified arguments. * @@ -118,6 +172,69 @@ class SteamContent : ClientMsgHandler() { return@async CDNAuthToken(message) } + /** + * TODO kdoc + * @param remoteClientId + * @param steamId + * @param serverRemoteClientId + * @param appId + * @param currentBuildId + * @return A [RequestPeerContentServer] + */ + fun requestPeerContentServer( + remoteClientId: Long, + steamId: Long, + serverRemoteClientId: Long, + appId: Int, + currentBuildId: Int = 0, + parentScope: CoroutineScope, + ): Deferred = parentScope.async { + val request = CContentServerDirectory_RequestPeerContentServer_Request.newBuilder().apply { + this.remoteClientId = remoteClientId + this.steamid = steamId + this.serverRemoteClientId = serverRemoteClientId + this.appId = appId + this.currentBuildId = currentBuildId + }.build() + + val message = contentService.requestPeerContentServer(request).await() + val response = message.body.build() + + return@async RequestPeerContentServer( + serverPort = response.serverPort, + installedDepots = response.installedDepotsList, + accessToken = response.accessToken, + ) + } + + /** + * TODO kdoc + * @param remoteClientId + * @param steamId + * @param serverRemoteClientId + * @return A [GetPeerContentInfo] + */ + fun getPeerContentInfo( + remoteClientId: Long, + steamId: Long, + serverRemoteClientId: Long, + parentScope: CoroutineScope, + ): Deferred = parentScope.async { + val request = CContentServerDirectory_GetPeerContentInfo_Request.newBuilder().apply { + this.remoteClientId = remoteClientId + this.steamid = steamId + this.serverRemoteClientId = serverRemoteClientId + }.build() + + val message = contentService.getPeerContentInfo(request).await() + val response = message.body.build() + + return@async GetPeerContentInfo( + appIds = response.appidsList, + ipPublic = response.ipPublic, + ) + } + /** * Handles a client message. This should not be called directly. * diff --git a/src/main/proto/in/dragonbra/javasteam/protobufs/webui/service_clientcomm.proto b/src/main/proto/in/dragonbra/javasteam/protobufs/webui/service_clientcomm.proto new file mode 100644 index 00000000..73c6ec45 --- /dev/null +++ b/src/main/proto/in/dragonbra/javasteam/protobufs/webui/service_clientcomm.proto @@ -0,0 +1,175 @@ +option java_package = "in.dragonbra.javasteam.protobufs.webui"; + +option optimize_for = SPEED; +option java_generic_services = false; + +message CClientComm_ClientData { + optional uint32 package_version = 1; + optional string os = 2; + optional string machine_name = 3; + optional string ip_public = 4; + optional string ip_private = 5; + optional uint64 bytes_available = 6; + repeated .CClientComm_ClientData_RunningGames running_games = 7; + optional uint32 protocol_version = 8; + optional uint32 clientcomm_version = 9; + repeated uint32 local_users = 10; +} + +message CClientComm_ClientData_RunningGames { + optional uint32 appid = 1; + optional string extra_info = 2; + optional uint32 time_running_sec = 3; +} + +message CClientComm_EnableOrDisableDownloads_Request { + optional uint64 client_instanceid = 1; + optional bool enable = 2; +} + +message CClientComm_EnableOrDisableDownloads_Response { +} + +message CClientComm_GetAllClientLogonInfo_Request { +} + +message CClientComm_GetAllClientLogonInfo_Response { + repeated .CClientComm_GetAllClientLogonInfo_Response_Session sessions = 1; + optional uint32 refetch_interval_sec = 2; +} + +message CClientComm_GetAllClientLogonInfo_Response_Session { + optional uint64 client_instanceid = 1; + optional uint32 protocol_version = 2; + optional string os_name = 3; + optional string machine_name = 4; + optional int32 os_type = 5; + optional int32 device_type = 6; + optional int32 realm = 7; +} + +message CClientComm_GetClientAppList_Request { + optional string fields = 1; + optional string filters = 2; + optional uint64 client_instanceid = 3; + optional bool include_client_info = 4; + optional string language = 5; + repeated uint32 filter_appids = 6; +} + +message CClientComm_GetClientAppList_Response { + optional uint64 bytes_available = 1; + repeated .CClientComm_GetClientAppList_Response_AppData apps = 2; + optional .CClientComm_ClientData client_info = 3; + optional uint32 refetch_interval_sec_full = 4; + optional uint32 refetch_interval_sec_changing = 5; + optional uint32 refetch_interval_sec_updating = 6; +} + +message CClientComm_GetClientAppList_Response_AppData { + optional uint32 appid = 1; + optional string app = 2; + optional string category = 3; + optional string app_type = 4; + optional uint32 num_downloading = 8; + optional uint32 bytes_download_rate = 11; + optional uint64 bytes_downloaded = 12; + optional uint64 bytes_to_download = 13; + repeated .CClientComm_GetClientAppList_Response_AppData_DLCData dlcs = 17; + optional bool favorite = 18; + optional bool auto_update = 19; + optional bool installed = 20; + optional bool download_paused = 21; + optional bool changing = 22; + optional bool available_on_platform = 23; + optional uint64 bytes_staged = 24; + optional uint64 bytes_to_stage = 25; + optional uint64 bytes_required = 26; + optional uint32 source_buildid = 27; + optional uint32 target_buildid = 28; + optional uint32 estimated_seconds_remaining = 29; + optional int32 queue_position = 30 [default = -1]; + optional bool uninstalling = 31; + optional uint32 rt_time_scheduled = 32; + optional bool running = 33; + optional uint32 update_percentage = 34; +} + +message CClientComm_GetClientAppList_Response_AppData_DLCData { + optional uint32 appid = 1; + optional string app = 2; + optional uint32 installed = 3; +} + +message CClientComm_GetClientInfo_Request { + optional uint64 client_instanceid = 1; +} + +message CClientComm_GetClientInfo_Response { + optional .CClientComm_ClientData client_info = 1; +} + +message CClientComm_GetClientLogonInfo_Request { + optional uint64 client_instanceid = 1; +} + +message CClientComm_GetClientLogonInfo_Response { + optional uint32 protocol_version = 1; + optional string os = 2; + optional string machine_name = 3; +} + +message CClientComm_InstallClientApp_Request { + optional uint32 appid = 1; + optional uint64 client_instanceid = 2; +} + +message CClientComm_InstallClientApp_Response { +} + +message CClientComm_LaunchClientApp_Request { + optional uint64 client_instanceid = 1; + optional uint32 appid = 2; + optional string query_params = 3; +} + +message CClientComm_LaunchClientApp_Response { +} + +message CClientComm_SetClientAppUpdateState_Request { + optional uint32 appid = 1; + optional uint32 action = 2; + optional uint64 client_instanceid = 3; +} + +message CClientComm_SetClientAppUpdateState_Response { +} + +message CClientComm_UninstallClientApp_Request { + optional uint32 appid = 1; + optional uint64 client_instanceid = 2; +} + +message CClientComm_UninstallClientApp_Response { +} + +service ClientComm { + // ePrivilege=1 + rpc EnableOrDisableDownloads (.CClientComm_EnableOrDisableDownloads_Request) returns (.CClientComm_EnableOrDisableDownloads_Response); + // bConstMethod=true, ePrivilege=1 + rpc GetAllClientLogonInfo (.CClientComm_GetAllClientLogonInfo_Request) returns (.CClientComm_GetAllClientLogonInfo_Response); + // bConstMethod=true, ePrivilege=1 + rpc GetClientAppList (.CClientComm_GetClientAppList_Request) returns (.CClientComm_GetClientAppList_Response); + // bConstMethod=true, ePrivilege=1 + rpc GetClientInfo (.CClientComm_GetClientInfo_Request) returns (.CClientComm_GetClientInfo_Response); + // bConstMethod=true, ePrivilege=1 + rpc GetClientLogonInfo (.CClientComm_GetClientLogonInfo_Request) returns (.CClientComm_GetClientLogonInfo_Response); + // ePrivilege=1 + rpc InstallClientApp (.CClientComm_InstallClientApp_Request) returns (.CClientComm_InstallClientApp_Response); + // ePrivilege=1 + rpc LaunchClientApp (.CClientComm_LaunchClientApp_Request) returns (.CClientComm_LaunchClientApp_Response); + // ePrivilege=1 + rpc SetClientAppUpdateState (.CClientComm_SetClientAppUpdateState_Request) returns (.CClientComm_SetClientAppUpdateState_Response); + // ePrivilege=1 + rpc UninstallClientApp (.CClientComm_UninstallClientApp_Request) returns (.CClientComm_UninstallClientApp_Response); +} diff --git a/src/test/java/in/dragonbra/javasteam/rpc/UnifiedInterfaceTest.kt b/src/test/java/in/dragonbra/javasteam/rpc/UnifiedInterfaceTest.kt index 1397fd3e..3fd245c4 100644 --- a/src/test/java/in/dragonbra/javasteam/rpc/UnifiedInterfaceTest.kt +++ b/src/test/java/in/dragonbra/javasteam/rpc/UnifiedInterfaceTest.kt @@ -80,6 +80,9 @@ class UnifiedInterfaceTest { "UserAccount.kt", "PublishedFile.kt", "PublishedFileClient.kt", + + // WebUI + "ClientComm.kt" ) } } From 5ca61593edfab40315d8f61eeb6781dcfd391b7b Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 17 Dec 2025 10:24:14 -0600 Subject: [PATCH 26/44] Add annotations to new code. --- .../steam/handlers/steamclientcommunication/ClientAppList.kt | 3 +++ .../steamclientcommunication/ClientAppListAppData.kt | 3 +++ .../steamclientcommunication/ClientAppListDlcData.kt | 3 +++ .../steam/handlers/steamclientcommunication/ClientInfo.kt | 3 +++ .../handlers/steamclientcommunication/ClientLogonInfo.kt | 3 +++ .../steamclientcommunication/ClientLogonInfoSession.kt | 2 ++ .../handlers/steamclientcommunication/InstalledAppsFilter.kt | 3 +++ .../steam/handlers/steamclientcommunication/RunningGames.kt | 3 +++ .../steamclientcommunication/SteamClientCommunication.kt | 2 ++ .../steam/handlers/steamcontent/ClientUpdateHosts.kt | 3 +++ .../javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt | 3 +++ .../steam/handlers/steamcontent/GetPeerContentInfo.kt | 3 +++ .../steam/handlers/steamcontent/RequestPeerContentServer.kt | 3 +++ .../javasteam/steam/handlers/steamcontent/SteamContent.kt | 5 +++++ .../in/dragonbra/javasteam/steam/steamclient/SteamClient.kt | 1 + 15 files changed, 43 insertions(+) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt index 2bbe8ecb..3d90e73f 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt @@ -1,5 +1,7 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param bytesAvailable @@ -9,6 +11,7 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication * @param refetchIntervalSecChanging * @param refetchIntervalSecUpdating */ +@JavaSteamAddition data class ClientAppList( val bytesAvailable: Long, val apps: List, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt index eb2ea9bd..187b7504 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt @@ -1,5 +1,7 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param appid @@ -29,6 +31,7 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication * @param running * @param updatePercentage */ +@JavaSteamAddition data class ClientAppListAppData( val appid: Int, val app: String, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt index 525246ea..97966fc7 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt @@ -1,11 +1,14 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param appId * @param app * @param installed */ +@JavaSteamAddition data class ClientAppListDlcData( val appId: Int, val app: String, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt index a7343f5e..905bbd4b 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt @@ -1,5 +1,7 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param packageVersion @@ -13,6 +15,7 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication * @param clientCommVersion * @param localUsers */ +@JavaSteamAddition data class ClientInfo( val packageVersion: Int, val os: String, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt index 16401061..4f507ef9 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt @@ -1,10 +1,13 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param sessions * @param refetchIntervalSec */ +@JavaSteamAddition data class ClientLogonInfo( val sessions: List, val refetchIntervalSec: Int, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt index fac90b8d..4e0fade9 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt @@ -3,6 +3,7 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication import `in`.dragonbra.javasteam.enums.EGamingDeviceType import `in`.dragonbra.javasteam.enums.EOSType import `in`.dragonbra.javasteam.enums.ESteamRealm +import `in`.dragonbra.javasteam.util.JavaSteamAddition /** * Describes an active device running desktop Steam client. @@ -14,6 +15,7 @@ import `in`.dragonbra.javasteam.enums.ESteamRealm * @param deviceType Type of the device, such as [EGamingDeviceType].k_EGamingDeviceType_SteamDeck * @param realm Realm of the session from. See [ESteamRealm]. */ +@JavaSteamAddition data class ClientLogonInfoSession( val clientInstanceId: Long, val protocolVersion: Int, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt index 5491c0fa..246e8fa5 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt @@ -1,5 +1,8 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication +import `in`.dragonbra.javasteam.util.JavaSteamAddition + +@JavaSteamAddition enum class InstalledAppsFilter { /** * Return everything, including not installed apps diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt index d04b97fd..6b1744fc 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt @@ -1,11 +1,14 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param appId * @param extraInfo * @param timeRunningSec */ +@JavaSteamAddition data class RunningGames( val appId: Int, val extraInfo: String, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt index 29732ad7..f8029067 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt @@ -12,12 +12,14 @@ import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_Un import `in`.dragonbra.javasteam.rpc.service.ClientComm import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages +import `in`.dragonbra.javasteam.util.JavaSteamAddition import kotlinx.coroutines.Deferred import kotlinx.coroutines.async /** * Allows controlling of other running Steam clients. */ +@JavaSteamAddition class SteamClientCommunication : ClientMsgHandler() { private val clientComm: ClientComm by lazy { diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt index 364dcee2..b1039846 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt @@ -1,11 +1,14 @@ package `in`.dragonbra.javasteam.steam.handlers.steamcontent +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param hostsKv * @param validUntilTime * @param ipCountry */ +@JavaSteamAddition data class ClientUpdateHosts( val hostsKv: String, val validUntilTime: Long, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt index 1ffb444e..03339599 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt @@ -1,11 +1,14 @@ package `in`.dragonbra.javasteam.steam.handlers.steamcontent +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param isAvailable * @param patchSize * @param patchedChunksSize */ +@JavaSteamAddition data class DepotPatchInfo( val isAvailable: Boolean, val patchSize: Long, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt index a6e125a7..f96187e2 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/GetPeerContentInfo.kt @@ -1,10 +1,13 @@ package `in`.dragonbra.javasteam.steam.handlers.steamcontent +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param appIds * @param ipPublic */ +@JavaSteamAddition data class GetPeerContentInfo( val appIds: List, val ipPublic: String, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt index 60644c32..72d46f9d 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt @@ -1,11 +1,14 @@ package `in`.dragonbra.javasteam.steam.handlers.steamcontent +import `in`.dragonbra.javasteam.util.JavaSteamAddition + /** * TODO kdoc * @param serverPort * @param installedDepots * @param accessToken */ +@JavaSteamAddition data class RequestPeerContentServer( val serverPort: Int, val installedDepots: List, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt index 073898f0..a4f4c60c 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt @@ -13,6 +13,7 @@ import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages import `in`.dragonbra.javasteam.steam.webapi.ContentServerDirectoryService +import `in`.dragonbra.javasteam.util.JavaSteamAddition import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Deferred import kotlinx.coroutines.async @@ -59,6 +60,7 @@ class SteamContent : ClientMsgHandler() { * @param targetManifestId * @return A [DepotPatchInfo] */ + @JavaSteamAddition fun getDepotPatchInfo( appId: Int, depotId: Int, @@ -88,6 +90,7 @@ class SteamContent : ClientMsgHandler() { * @param cachedSignature * @return A [ClientUpdateHosts] */ + @JavaSteamAddition fun getClientUpdateHosts( cachedSignature: String, parentScope: CoroutineScope, @@ -181,6 +184,7 @@ class SteamContent : ClientMsgHandler() { * @param currentBuildId * @return A [RequestPeerContentServer] */ + @JavaSteamAddition fun requestPeerContentServer( remoteClientId: Long, steamId: Long, @@ -214,6 +218,7 @@ class SteamContent : ClientMsgHandler() { * @param serverRemoteClientId * @return A [GetPeerContentInfo] */ + @JavaSteamAddition fun getPeerContentInfo( remoteClientId: Long, steamId: Long, diff --git a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt index f4ab15dc..31a2db15 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/steamclient/SteamClient.kt @@ -85,6 +85,7 @@ class SteamClient @JvmOverloads constructor( addHandlerCore(SteamContent()) addHandlerCore(SteamAuthTicket()) addHandlerCore(SteamNotifications()) // JavaSteam Addition + // addHandlerCore(SteamClientCommunication()) // JavaSteam Addition, not enabled by default if (handlers.size != HANDLERS_COUNT) { logger.error("Handlers size didnt match handlers count (${handlers.size}) when initializing") From c2b8dbdf5414042fe5df0a2210b2b77b69c62625 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 17 Dec 2025 15:43:43 -0600 Subject: [PATCH 27/44] Complete todos, add and revice kdoc --- .../AllClientLogonInfo.kt | 21 +++ ...ession.kt => AllClientLogonInfoSession.kt} | 26 +-- .../steamclientcommunication/ClientAppList.kt | 14 +- .../ClientAppListAppData.kt | 58 +++---- .../ClientAppListDlcData.kt | 8 +- .../steamclientcommunication/ClientInfo.kt | 22 +-- .../ClientLogonInfo.kt | 24 +-- .../InstalledAppsFilter.kt | 2 +- .../steamclientcommunication/RunningGames.kt | 8 +- .../SteamClientCommunication.kt | 161 +++++++++++++----- 10 files changed, 214 insertions(+), 130 deletions(-) create mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/AllClientLogonInfo.kt rename src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/{ClientLogonInfoSession.kt => AllClientLogonInfoSession.kt} (57%) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/AllClientLogonInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/AllClientLogonInfo.kt new file mode 100644 index 00000000..5094a2d9 --- /dev/null +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/AllClientLogonInfo.kt @@ -0,0 +1,21 @@ +package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication + +import `in`.dragonbra.javasteam.util.JavaSteamAddition + +/** + * Information about all active Steam clients on the network. + * @param sessions A list of active client sessions with their logon information. + * @param refetchIntervalSec The recommended interval in seconds to refetch this data. + */ +@JavaSteamAddition +data class AllClientLogonInfo( + val sessions: List, + val refetchIntervalSec: Int, +) { + override fun toString(): String = """ + AllClientLogonInfo( + sessions=$sessions, + refetchIntervalSec=$refetchIntervalSec + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/AllClientLogonInfoSession.kt similarity index 57% rename from src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt rename to src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/AllClientLogonInfoSession.kt index 4e0fade9..a330fbf3 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfoSession.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/AllClientLogonInfoSession.kt @@ -6,27 +6,27 @@ import `in`.dragonbra.javasteam.enums.ESteamRealm import `in`.dragonbra.javasteam.util.JavaSteamAddition /** - * Describes an active device running desktop Steam client. - * @param clientInstanceId Instance ID that is used to send remote signals - * @param protocolVersion Protocol version of the client - * @param osName Name of installed OS, such as "Linux 6.x" - * @param machineName Name of the device, such as "steamdeck" - * @param osType Type of installed OS, such as [EOSType].k_Linux6x - * @param deviceType Type of the device, such as [EGamingDeviceType].k_EGamingDeviceType_SteamDeck - * @param realm Realm of the session from. See [ESteamRealm]. + * Describes an active device running a Steam client. + * @param clientInstanceId Instance ID used to send remote signals to this client. + * @param protocolVersion Protocol version of the client. + * @param osName Name of the installed OS, such as "Linux 6.x". + * @param machineName Name of the device, such as "steamdeck". + * @param osType Type of installed OS, such as [EOSType].Linux6x. + * @param deviceType Type of the device, such as [EGamingDeviceType].StandardPC. + * @param realm Realm of the session. See [ESteamRealm]. */ @JavaSteamAddition -data class ClientLogonInfoSession( +data class AllClientLogonInfoSession( val clientInstanceId: Long, val protocolVersion: Int, val osName: String, val machineName: String, - val osType: Int, - val deviceType: Int, - val realm: Int, + val osType: EOSType, + val deviceType: EGamingDeviceType, + val realm: ESteamRealm, ) { override fun toString(): String = """ - ClientLogonInfoSession( + AllClientLogonInfoSession( clientInstanceId=$clientInstanceId, protocolVersion=$protocolVersion, osName='$osName', diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt index 3d90e73f..01fbd8a5 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppList.kt @@ -3,13 +3,13 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication import `in`.dragonbra.javasteam.util.JavaSteamAddition /** - * TODO kdoc - * @param bytesAvailable - * @param apps - * @param clientInfo - * @param refetchIntervalSecFull - * @param refetchIntervalSecChanging - * @param refetchIntervalSecUpdating + * Response containing the list of applications on a Steam client and related metadata. + * @param bytesAvailable Total bytes of storage available on the client. + * @param apps List of application data for each app on the client. + * @param clientInfo Information about the client machine and state. + * @param refetchIntervalSecFull Recommended interval in seconds to refetch the complete app list. + * @param refetchIntervalSecChanging Recommended interval in seconds to refetch when apps are changing state. + * @param refetchIntervalSecUpdating Recommended interval in seconds to refetch when apps are updating. */ @JavaSteamAddition data class ClientAppList( diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt index 187b7504..4af2ed7f 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListAppData.kt @@ -3,33 +3,33 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication import `in`.dragonbra.javasteam.util.JavaSteamAddition /** - * TODO kdoc - * @param appid - * @param app - * @param category - * @param appType - * @param numDownloading - * @param bytesDownloadRate - * @param bytesDownloaded - * @param bytesToDownload - * @param dlcs - * @param favorite - * @param autoUpdate - * @param installed - * @param downloadPaused - * @param changing - * @param availableOnPlatform - * @param bytesStaged - * @param bytesToStage - * @param bytesRequired - * @param sourceBuildId - * @param targetBuildId - * @param estimatedSecondsRemaining - * @param queuePosition - * @param uninstalling - * @param rtTimeScheduled - * @param running - * @param updatePercentage + * Detailed information about an application on a Steam client, including its download/update state. + * @param appid The application ID. + * @param app The application name. + * @param category The application category. + * @param appType The type of application (e.g., game, tool, DLC). + * @param numDownloading Number of items currently downloading for this app. + * @param bytesDownloadRate Current download speed in bytes per second. + * @param bytesDownloaded Total bytes already downloaded. + * @param bytesToDownload Total bytes that need to be downloaded. + * @param dlcs List of DLC data associated with this application. + * @param favorite Whether this app is marked as a favorite. + * @param autoUpdate Whether automatic updates are enabled for this app. + * @param installed Whether the app is currently installed. + * @param downloadPaused Whether the download is currently paused. + * @param changing Whether the app is currently changing state (installing/updating/uninstalling). + * @param availableOnPlatform Whether the app is available on the current platform. + * @param bytesStaged Bytes that have been staged for installation. + * @param bytesToStage Total bytes that need to be staged. + * @param bytesRequired Total disk space required for the app. + * @param sourceBuildId The current build ID installed. + * @param targetBuildId The build ID being updated to. + * @param estimatedSecondsRemaining Estimated time remaining for the current operation in seconds. + * @param queuePosition Position in the download queue (-1 if not queued). + * @param uninstalling Whether the app is currently being uninstalled. + * @param rtTimeScheduled Scheduled time for the update/download (Unix timestamp). + * @param running Whether the app is currently running. + * @param updatePercentage Progress percentage of the current update/download operation. */ @JavaSteamAddition data class ClientAppListAppData( @@ -63,8 +63,8 @@ data class ClientAppListAppData( override fun toString(): String = """ ClientAppListAppData( appid=$appid, - app='$app', c - ategory='$category', + app='$app', + category='$category', appType='$appType', numDownloading=$numDownloading, bytesDownloadRate=$bytesDownloadRate, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt index 97966fc7..631fcc58 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientAppListDlcData.kt @@ -3,10 +3,10 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication import `in`.dragonbra.javasteam.util.JavaSteamAddition /** - * TODO kdoc - * @param appId - * @param app - * @param installed + * Information about a DLC (Downloadable Content) associated with an application. + * @param appId The DLC's application ID. + * @param app The DLC's name. + * @param installed Installation status (0 = not installed, non-zero = installed). */ @JavaSteamAddition data class ClientAppListDlcData( diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt index 905bbd4b..b391fbf2 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientInfo.kt @@ -3,17 +3,17 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication import `in`.dragonbra.javasteam.util.JavaSteamAddition /** - * TODO kdoc - * @param packageVersion - * @param os - * @param machineName - * @param ipPublic - * @param ipPrivate - * @param bytesAvailable - * @param runningGames - * @param protocolVersion - * @param clientCommVersion - * @param localUsers + * Information about a Steam client machine and its current state. + * @param packageVersion The Steam client package version. + * @param os The operating system of the client. + * @param machineName The name of the client machine. + * @param ipPublic The client's public IP address. + * @param ipPrivate The client's private/local IP address. + * @param bytesAvailable Available disk space in bytes. + * @param runningGames List of games currently running on the client. + * @param protocolVersion The protocol version being used. + * @param clientCommVersion The client communication protocol version. + * @param localUsers List of local user IDs logged into the client. */ @JavaSteamAddition data class ClientInfo( diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt index 4f507ef9..95fcdb69 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/ClientLogonInfo.kt @@ -1,21 +1,13 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication -import `in`.dragonbra.javasteam.util.JavaSteamAddition - /** - * TODO kdoc - * @param sessions - * @param refetchIntervalSec + * Logon information for a specific Steam client. + * @param protocolVersion Protocol version of the client. + * @param os Operating system name. + * @param machineName Name of the client machine. */ -@JavaSteamAddition data class ClientLogonInfo( - val sessions: List, - val refetchIntervalSec: Int, -) { - override fun toString(): String = """ - ClientLogonInfo( - sessions=$sessions, - refetchIntervalSec=$refetchIntervalSec - ) - """.trimIndent() -} + val protocolVersion: Int, + val os: String, + val machineName: String, +) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt index 246e8fa5..47306ae6 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/InstalledAppsFilter.kt @@ -10,7 +10,7 @@ enum class InstalledAppsFilter { None, /** - * Return only apps that are "in progress" - downloading, updated, scheduled + * Return only apps that are "in progress" - downloading, updating, scheduled */ Changing, } diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt index 6b1744fc..5274a34b 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/RunningGames.kt @@ -3,10 +3,10 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication import `in`.dragonbra.javasteam.util.JavaSteamAddition /** - * TODO kdoc - * @param appId - * @param extraInfo - * @param timeRunningSec + * Information about a game currently running on a Steam client. + * @param appId The application ID of the running game. + * @param extraInfo Additional information about the running game. + * @param timeRunningSec Duration in seconds that the game has been running. */ @JavaSteamAddition data class RunningGames( diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt index f8029067..deb5c795 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamclientcommunication/SteamClientCommunication.kt @@ -1,10 +1,15 @@ package `in`.dragonbra.javasteam.steam.handlers.steamclientcommunication import `in`.dragonbra.javasteam.base.IPacketMsg +import `in`.dragonbra.javasteam.enums.EGamingDeviceType +import `in`.dragonbra.javasteam.enums.EOSType import `in`.dragonbra.javasteam.enums.EResult +import `in`.dragonbra.javasteam.enums.ESteamRealm import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_EnableOrDisableDownloads_Request import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_GetAllClientLogonInfo_Request import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_GetClientAppList_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_GetClientInfo_Request +import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_GetClientLogonInfo_Request import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_InstallClientApp_Request import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_LaunchClientApp_Request import `in`.dragonbra.javasteam.protobufs.webui.ServiceClientcomm.CClientComm_SetClientAppUpdateState_Request @@ -19,6 +24,7 @@ import kotlinx.coroutines.async /** * Allows controlling of other running Steam clients. */ +@Suppress("unused") @JavaSteamAddition class SteamClientCommunication : ClientMsgHandler() { @@ -29,25 +35,26 @@ class SteamClientCommunication : ClientMsgHandler() { } /** - * Return the list of active devices that are running any Steam client and connected to the network. - * Note: this will return all connected clients. Filter the results based on OS or device type. + * Retrieves information about all active Steam clients connected to the network. + * Note: This returns all connected clients. Filter the results based on OS or device type as needed. + * @return Information about all active client sessions with recommended refetch interval. */ - fun getAllClientLogonInfo(): Deferred = client.defaultScope.async { + fun getAllClientLogonInfo(): Deferred = client.defaultScope.async { val request = CClientComm_GetAllClientLogonInfo_Request.newBuilder().build() val message = clientComm.getAllClientLogonInfo(request).await() val response = message.body.build() - return@async ClientLogonInfo( + return@async AllClientLogonInfo( sessions = response.sessionsList.map { - ClientLogonInfoSession( + AllClientLogonInfoSession( clientInstanceId = it.clientInstanceid, protocolVersion = it.protocolVersion, osName = it.osName, machineName = it.machineName, - osType = it.osType, - deviceType = it.deviceType, - realm = it.realm, + osType = EOSType.from(it.osType), + deviceType = EGamingDeviceType.from(it.deviceType), + realm = ESteamRealm.from(it.realm), ) }, refetchIntervalSec = response.refetchIntervalSec, @@ -57,9 +64,9 @@ class SteamClientCommunication : ClientMsgHandler() { /** * Return the list of applications of the remote device. * This is not the list of downloaded apps, but the whole "available" list with a flag indicating about it being downloaded. - * @param remoteId remote session ID - * @param filters filters - * @param language language + * @param remoteId The remote session ID of the client to query. + * @param filters filters to choose, see [InstalledAppsFilter]. + * @param language language for localized app names (e.g., "english", "french"). */ @JvmOverloads fun getClientAppList( @@ -145,9 +152,9 @@ class SteamClientCommunication : ClientMsgHandler() { /** * Adds the application to the remote installation queue. - * @param remoteId - * @param appId - * @return **true** if successfully, otherwise false. + * @param remoteId The remote session ID of the client to query. + * @param appId Application ID to install. + * @return **true** if successful, otherwise false. */ fun installClientApp(remoteId: Long, appId: Int): Deferred = client.defaultScope.async { val request = CClientComm_InstallClientApp_Request.newBuilder().apply { @@ -164,12 +171,17 @@ class SteamClientCommunication : ClientMsgHandler() { /** * Sets the update state of an app in remote installation queue. * Action set to true will move the requested app to the top of the queue. - * @param remoteId - * @param appId - * @param action - * @return **true** if successfully, otherwise false. + * @param remoteId The remote session ID of the client to query + * @param appId Application ID to update. + * @param action true to prioritize (move to top of queue), false otherwise. + * @return **true** if successful, otherwise false. */ - fun setClientAppUpdateState(remoteId: Long, appId: Int, action: Boolean): Deferred = + @JvmOverloads + fun setClientAppUpdateState( + remoteId: Long, + appId: Int, + action: Boolean = false, + ): Deferred = client.defaultScope.async { val request = CClientComm_SetClientAppUpdateState_Request.newBuilder().apply { this.clientInstanceid = remoteId @@ -185,11 +197,14 @@ class SteamClientCommunication : ClientMsgHandler() { /** * Requests to uninstall the app from the device. - * @param remoteId - * @param appId - * @return **true** if successfully, otherwise false. + * @param remoteId The remote session ID of the client to query. + * @param appId Application ID to uninstall. + * @return **true** if successful, otherwise false. */ - fun uninstallClientApp(remoteId: Long, appId: Int): Deferred = client.defaultScope.async { + fun uninstallClientApp( + remoteId: Long, + appId: Int, + ): Deferred = client.defaultScope.async { val request = CClientComm_UninstallClientApp_Request.newBuilder().apply { this.clientInstanceid = remoteId this.appid = appId @@ -202,12 +217,15 @@ class SteamClientCommunication : ClientMsgHandler() { } /** - * Pauses or resumes active download - the first item in the queue. - * @param remoteId - * @param enable - * @return **true** if successfully, otherwise false. + * Pauses or resumes downloads on the remote client. + * @param remoteId The remote session ID of the client to query. + * @param enable true to resume downloads, false to pause. + * @return **true** if successful, otherwise false. */ - fun enableOrDisableDownloads(remoteId: Long, enable: Boolean): Deferred = client.defaultScope.async { + fun enableOrDisableDownloads( + remoteId: Long, + enable: Boolean, + ): Deferred = client.defaultScope.async { val request = CClientComm_EnableOrDisableDownloads_Request.newBuilder().apply { this.clientInstanceid = remoteId this.enable = enable @@ -221,28 +239,81 @@ class SteamClientCommunication : ClientMsgHandler() { /** * Launches the application on the remote device. - * @param remoteId - * @param appId - * @param parameters - * @return **true** if successfully, otherwise false. + * @param remoteId The remote session ID of the client to query. + * @param appId application ID to launch. + * @param parameters Optional launch parameters/query string (e.g., command line arguments). + * @return **true** if successful, otherwise false. */ - fun launchClientApp(remoteId: Long, appId: Int, parameters: String? = null): Deferred = - client.defaultScope.async { - val request = CClientComm_LaunchClientApp_Request.newBuilder().apply { - this.clientInstanceid = remoteId - this.appid = appId - this.queryParams = parameters.orEmpty() - }.build() + @JvmOverloads + fun launchClientApp( + remoteId: Long, + appId: Int, + parameters: String? = null, + ): Deferred = client.defaultScope.async { + val request = CClientComm_LaunchClientApp_Request.newBuilder().apply { + this.clientInstanceid = remoteId + this.appid = appId + this.queryParams = parameters.orEmpty() + }.build() - val message = clientComm.launchClientApp(request).await() - // Request has empty response. + val message = clientComm.launchClientApp(request).await() + // Request has empty response. - return@async message.result == EResult.OK - } + return@async message.result == EResult.OK + } - // TODO: GetClientLogonInfo() + /** + * Retrieves logon information for a specific Steam client. + * @param remoteId The remote session ID of the client to query. + * @return Client logon information including protocol version, OS, and machine name. + */ + fun getClientLogonInfo(remoteId: Long): Deferred = client.defaultScope.async { + val request = CClientComm_GetClientLogonInfo_Request.newBuilder().apply { + this.clientInstanceid = remoteId + }.build() + + val message = clientComm.getClientLogonInfo(request).await() + val response = message.body.build() + + return@async ClientLogonInfo( + protocolVersion = response.protocolVersion, + os = response.os, + machineName = response.machineName + ) + } + + /** + * Retrieves detailed information about a specific Steam client, including system info and running games. + * @param remoteId The remote session ID of the client to query. + * @return Detailed client information including hardware, network, and active games. + */ + fun getClientInfo(remoteId: Long): Deferred = client.defaultScope.async { + val request = CClientComm_GetClientInfo_Request.newBuilder().apply { + this.clientInstanceid = remoteId + }.build() + + val message = clientComm.getClientInfo(request).await() + val response = message.body.build() - // TODO: GetClientInfo() + return@async ClientInfo( + packageVersion = response.clientInfo.packageVersion, + os = response.clientInfo.os, + machineName = response.clientInfo.machineName, + ipPublic = response.clientInfo.ipPublic, + ipPrivate = response.clientInfo.ipPrivate, + bytesAvailable = response.clientInfo.bytesAvailable, + runningGames = response.clientInfo.runningGamesList.map { game -> + RunningGames( + appId = game.appid, + extraInfo = game.extraInfo, + timeRunningSec = game.timeRunningSec, + ) + }, + protocolVersion = response.clientInfo.protocolVersion, + clientCommVersion = response.clientInfo.clientcommVersion, + localUsers = response.clientInfo.localUsersList, + ) + } /** * Handles a client message. This should not be called directly. From 6a947c455a0336a6494449f13bde505f9c20805c Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 17 Dec 2025 16:14:48 -0600 Subject: [PATCH 28/44] Add kdoc to SteamContent.getDepotPatchInfo --- .../handlers/steamcontent/DepotPatchInfo.kt | 12 +++++++----- .../steam/handlers/steamcontent/SteamContent.kt | 16 +++++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt index 03339599..693af07e 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/DepotPatchInfo.kt @@ -3,14 +3,16 @@ package `in`.dragonbra.javasteam.steam.handlers.steamcontent import `in`.dragonbra.javasteam.util.JavaSteamAddition /** - * TODO kdoc - * @param isAvailable - * @param patchSize - * @param patchedChunksSize + * Information about a depot patch between two manifest versions. + * @param isAvailable Whether a patch is available for this upgrade path. + * @param patchSize Size of the patch file in bytes (delta to download). + * @param patchedChunksSize Total size in bytes of content that will be patched/modified. */ @JavaSteamAddition data class DepotPatchInfo( val isAvailable: Boolean, val patchSize: Long, val patchedChunksSize: Long, -) +) { + override fun toString(): String = "DepotPatchInfo(isAvailable=$isAvailable, patchSize=$patchSize, patchedChunksSize=$patchedChunksSize)" +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt index a4f4c60c..701a647b 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt @@ -53,20 +53,22 @@ class SteamContent : ClientMsgHandler() { } /** - * TODO kdoc - * @param appId - * @param depotId - * @param sourceManifestId - * @param targetManifestId - * @return A [DepotPatchInfo] + * Retrieves patch information for upgrading a depot from one manifest version to another. + * @param appId The application ID. + * @param depotId The depot ID to get patch info for. + * @param sourceManifestId The current manifest ID to upgrade from. + * @param targetManifestId The target manifest ID to upgrade to. + * @param parentScope Coroutine scope for the async operation. + * @return A [DepotPatchInfo] containing patch availability, patch file size, and total patched content size. */ @JavaSteamAddition + @JvmOverloads fun getDepotPatchInfo( appId: Int, depotId: Int, sourceManifestId: Long, targetManifestId: Long, - parentScope: CoroutineScope, + parentScope: CoroutineScope = client.defaultScope, ): Deferred = parentScope.async { val request = CContentServerDirectory_GetDepotPatchInfo_Request.newBuilder().apply { this.appid = appId From e8afafc0914956278ac66087cb10bc39b6b9a438 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 17 Dec 2025 23:40:07 -0600 Subject: [PATCH 29/44] add clientInstanceId to login response, other tweaks. --- .../steamcontent/ClientUpdateHosts.kt | 16 ------- .../steamcontent/RequestPeerContentServer.kt | 10 +++- .../handlers/steamcontent/SteamContent.kt | 48 ++++++++++--------- .../steamuser/callback/LoggedOnCallback.kt | 9 ++++ 4 files changed, 44 insertions(+), 39 deletions(-) delete mode 100644 src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt deleted file mode 100644 index b1039846..00000000 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/ClientUpdateHosts.kt +++ /dev/null @@ -1,16 +0,0 @@ -package `in`.dragonbra.javasteam.steam.handlers.steamcontent - -import `in`.dragonbra.javasteam.util.JavaSteamAddition - -/** - * TODO kdoc - * @param hostsKv - * @param validUntilTime - * @param ipCountry - */ -@JavaSteamAddition -data class ClientUpdateHosts( - val hostsKv: String, - val validUntilTime: Long, - val ipCountry: String, -) diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt index 72d46f9d..b0438d37 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/RequestPeerContentServer.kt @@ -13,4 +13,12 @@ data class RequestPeerContentServer( val serverPort: Int, val installedDepots: List, val accessToken: Long, -) +) { + override fun toString(): String = """ + RequestPeerContentServer( + serverPort=$serverPort, + installedDepots=$installedDepots, + accessToken=$accessToken + ) + """.trimIndent() +} diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt index 701a647b..7d2cb71d 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamcontent/SteamContent.kt @@ -1,8 +1,8 @@ package `in`.dragonbra.javasteam.steam.handlers.steamcontent import `in`.dragonbra.javasteam.base.IPacketMsg +import `in`.dragonbra.javasteam.enums.EResult import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetCDNAuthToken_Request -import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetClientUpdateHosts_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetDepotPatchInfo_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetManifestRequestCode_Request import `in`.dragonbra.javasteam.protobufs.steamclient.SteammessagesContentsystemSteamclient.CContentServerDirectory_GetPeerContentInfo_Request @@ -14,6 +14,8 @@ import `in`.dragonbra.javasteam.steam.handlers.ClientMsgHandler import `in`.dragonbra.javasteam.steam.handlers.steamunifiedmessages.SteamUnifiedMessages import `in`.dragonbra.javasteam.steam.webapi.ContentServerDirectoryService import `in`.dragonbra.javasteam.util.JavaSteamAddition +import `in`.dragonbra.javasteam.util.log.LogManager +import `in`.dragonbra.javasteam.util.log.Logger import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Deferred import kotlinx.coroutines.async @@ -21,7 +23,13 @@ import kotlinx.coroutines.async /** * This handler is used for interacting with content server directory on the Steam network. */ +@Suppress("unused") class SteamContent : ClientMsgHandler() { + + companion object { + private val logger: Logger = LogManager.getLogger(SteamContent::class.java) + } + private val contentService: ContentServerDirectory by lazy { val unifiedMessages = client.getHandler(SteamUnifiedMessages::class.java) ?: throw NullPointerException("Unable to get SteamUnifiedMessages handler") @@ -80,6 +88,10 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getDepotPatchInfo(request).await() val response = message.body.build() + if (message.result != EResult.OK) { + logger.error("getDepotPatchInfo got ${message.result}") + } + return@async DepotPatchInfo( isAvailable = response.isAvailable, patchSize = response.patchSize, @@ -87,25 +99,7 @@ class SteamContent : ClientMsgHandler() { ) } - /** - * TODO kdoc - * @param cachedSignature - * @return A [ClientUpdateHosts] - */ - @JavaSteamAddition - fun getClientUpdateHosts( - cachedSignature: String, - parentScope: CoroutineScope, - ): Deferred = parentScope.async { - val request = CContentServerDirectory_GetClientUpdateHosts_Request.newBuilder().apply { - this.cachedSignature = cachedSignature - }.build() - - val message = contentService.getClientUpdateHosts(request).await() - val response = message.body.build() - - return@async ClientUpdateHosts(response.hostsKv, response.validUntilTime, response.ipCountry) - } + // RPC getClientUpdateHosts() is not applicable for JavaSteam. This is used for Steam's own updating stuff. /** * Request the manifest request code for the specified arguments. @@ -187,13 +181,14 @@ class SteamContent : ClientMsgHandler() { * @return A [RequestPeerContentServer] */ @JavaSteamAddition + @JvmOverloads fun requestPeerContentServer( remoteClientId: Long, steamId: Long, serverRemoteClientId: Long, appId: Int, currentBuildId: Int = 0, - parentScope: CoroutineScope, + parentScope: CoroutineScope = client.defaultScope, ): Deferred = parentScope.async { val request = CContentServerDirectory_RequestPeerContentServer_Request.newBuilder().apply { this.remoteClientId = remoteClientId @@ -206,6 +201,10 @@ class SteamContent : ClientMsgHandler() { val message = contentService.requestPeerContentServer(request).await() val response = message.body.build() + if (message.result != EResult.OK) { + logger.error("requestPeerContentServer got ${message.result}") + } + return@async RequestPeerContentServer( serverPort = response.serverPort, installedDepots = response.installedDepotsList, @@ -221,11 +220,12 @@ class SteamContent : ClientMsgHandler() { * @return A [GetPeerContentInfo] */ @JavaSteamAddition + @JvmOverloads fun getPeerContentInfo( remoteClientId: Long, steamId: Long, serverRemoteClientId: Long, - parentScope: CoroutineScope, + parentScope: CoroutineScope = client.defaultScope, ): Deferred = parentScope.async { val request = CContentServerDirectory_GetPeerContentInfo_Request.newBuilder().apply { this.remoteClientId = remoteClientId @@ -236,6 +236,10 @@ class SteamContent : ClientMsgHandler() { val message = contentService.getPeerContentInfo(request).await() val response = message.body.build() + if (message.result != EResult.OK) { + logger.error("getPeerContentInfo got ${message.result}") + } + return@async GetPeerContentInfo( appIds = response.appidsList, ipPublic = response.ipPublic, diff --git a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamuser/callback/LoggedOnCallback.kt b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamuser/callback/LoggedOnCallback.kt index 66b21e4b..84ab0712 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/handlers/steamuser/callback/LoggedOnCallback.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/handlers/steamuser/callback/LoggedOnCallback.kt @@ -136,6 +136,13 @@ class LoggedOnCallback : CallbackMsg { var familyGroupId: Long = 0L private set + /** + * Gets the client instance ID. + * This is used for P2P content sharing operations. + */ + var clientInstanceId: Long = 0L + private set + constructor(packetMsg: IPacketMsg) { if (!packetMsg.isProto) { handleNonProtoLogon(packetMsg) @@ -185,6 +192,8 @@ class LoggedOnCallback : CallbackMsg { } familyGroupId = resp.familyGroupId + + clientInstanceId = resp.clientInstanceId } constructor(result: EResult) { From 7eba1c7d38e96ab07a713dd13577f3a99fca1d3d Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Tue, 9 Dec 2025 10:30:38 +0800 Subject: [PATCH 30/44] Add maxFileWrites, separate download and file write operation to improve network utilization --- .../depotdownloader/DepotDownloader.kt | 200 +++++++++++------- 1 file changed, 126 insertions(+), 74 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index c9492a8d..e34e5b66 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -106,6 +106,7 @@ class DepotDownloader @JvmOverloads constructor( private val debug: Boolean = false, private val useLanCache: Boolean = false, private var maxDownloads: Int = 8, + private var maxFileWrites: Int = 8, private val androidEmulation: Boolean = false, ) : Closeable { @@ -150,6 +151,8 @@ class DepotDownloader @JvmOverloads constructor( private var processingChannel = Channel(Channel.UNLIMITED) + private var fileWriteChannel = Channel(Channel.UNLIMITED) + private var steam3: Steam3Session? = null // region [REGION] Private data classes. @@ -161,6 +164,17 @@ class DepotDownloader @JvmOverloads constructor( val totalChunksForFile: Int, ) + private data class FileWriteItem( + val depot: DepotDownloadInfo, + val depotDownloadCounter: DepotDownloadCounter, + val downloadCounter: GlobalDownloadCounter, + val written: Int, + val file: FileData, + val fileStreamData: FileStreamData, + val chunk: ChunkData, + val chunkBuffer: ByteArray, + ) + private data class DirectoryResult(val success: Boolean, val installDir: Path?) private data class Config( @@ -200,6 +214,11 @@ class DepotDownloader @JvmOverloads constructor( scope.launch { processItems() } + + // Launch the file writing loop + scope.launch { + processFileWrites() + } } // region [REGION] Downloading Operations @@ -1456,80 +1475,19 @@ class DepotDownloader @JvmOverloads constructor( // Throw the cancellation exception if requested so that this task is marked failed ensureActive() - try { - fileStreamData.fileLock.lock() - - if (fileStreamData.fileHandle == null) { - val fileFinalPath = depot.installDir / file.fileName - fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) - } - - fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) - } finally { - fileStreamData.fileLock.unlock() - } - - val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() - if (remainingChunks == 0) { - fileStreamData.fileHandle?.close() - - // File completed - notify with percentage - val sizeDownloaded = synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += written.toLong() - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - depotDownloadCounter.sizeDownloaded - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - - val fileFinalPath = depot.installDir / file.fileName - val depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) - - notifyListeners { listener -> - listener.onFileCompleted( - depotId = depot.depotId, - fileName = fileFinalPath.toString(), - depotPercentComplete = depotPercentage - ) - } - - logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) - } else { - // Update counters and notify on chunk completion - val sizeDownloaded: Long - val depotPercentage: Float - val compressedBytes: Long - val uncompressedBytes: Long - - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += written.toLong() - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - - sizeDownloaded = depotDownloadCounter.sizeDownloaded - compressedBytes = depotDownloadCounter.depotBytesCompressed - uncompressedBytes = depotDownloadCounter.depotBytesUncompressed - depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) - } - - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } - - notifyListeners { listener -> - listener.onChunkCompleted( - depotId = depot.depotId, - depotPercentComplete = depotPercentage, - compressedBytes = compressedBytes, - uncompressedBytes = uncompressedBytes - ) - } - } + // Queue the file write operation + fileWriteChannel.send( + FileWriteItem( + depot = depot, + depotDownloadCounter = depotDownloadCounter, + downloadCounter = downloadCounter, + written = written, + file = file, + fileStreamData = fileStreamData, + chunk = chunk, + chunkBuffer = chunkBuffer, + ) + ) } private fun testIsFileIncluded(filename: String): Boolean { @@ -1757,4 +1715,98 @@ class DepotDownloader @JvmOverloads constructor( logger = null } + + private suspend fun processFileWrites() = coroutineScope { + while (!completionFuture.isDone && !completionFuture.isCancelled && !completionFuture.isCompletedExceptionally) { + List(maxFileWrites) { + async { + for (item in fileWriteChannel) { + val depot = item.depot + val depotDownloadCounter = item.depotDownloadCounter + val downloadCounter = item.downloadCounter + val written = item.written + val file = item.file + val fileStreamData = item.fileStreamData + val chunk = item.chunk + val chunkBuffer = item.chunkBuffer + + try { + fileStreamData.fileLock.lock() + + if (fileStreamData.fileHandle == null) { + val fileFinalPath = depot.installDir / file.fileName + fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) + } + + fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) + } finally { + fileStreamData.fileLock.unlock() + } + + val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() + if (remainingChunks == 0) { + fileStreamData.fileHandle?.close() + + // File completed - notify with percentage + val sizeDownloaded = synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + depotDownloadCounter.sizeDownloaded + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + + val fileFinalPath = depot.installDir / file.fileName + val depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) + + notifyListeners { listener -> + listener.onFileCompleted( + depotId = depot.depotId, + fileName = fileFinalPath.toString(), + depotPercentComplete = depotPercentage + ) + } + + logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) + } else { + // Update counters and notify on chunk completion + val sizeDownloaded: Long + val depotPercentage: Float + val compressedBytes: Long + val uncompressedBytes: Long + + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + + sizeDownloaded = depotDownloadCounter.sizeDownloaded + compressedBytes = depotDownloadCounter.depotBytesCompressed + uncompressedBytes = depotDownloadCounter.depotBytesUncompressed + depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) + } + + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + + notifyListeners { listener -> + listener.onChunkCompleted( + depotId = depot.depotId, + depotPercentComplete = depotPercentage, + compressedBytes = compressedBytes, + uncompressedBytes = uncompressedBytes + ) + } + } + } + } + }.awaitAll() + } + } } From f747e4ec5c1c066125feb8e9deb742ee5fc2e9dd Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Tue, 9 Dec 2025 17:37:10 +0800 Subject: [PATCH 31/44] change notifyListeners to use coroutineScope for non blocking call to update UI --- .../dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index e34e5b66..43857933 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -1522,8 +1522,10 @@ class DepotDownloader @JvmOverloads constructor( listeners.remove(listener) } - private fun notifyListeners(action: (IDownloadListener) -> Unit) { - listeners.forEach { listener -> action(listener) } + private suspend fun notifyListeners(action: (IDownloadListener) -> Unit) { + coroutineScope { + listeners.forEach { listener -> action(listener) } + } } // endregion From c4d3d4ce10710b37a15ef288863a5bc3e4999529 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Wed, 10 Dec 2025 08:57:48 +0800 Subject: [PATCH 32/44] changed to use pre-launch file workers for better I/O performance --- .../depotdownloader/DepotDownloader.kt | 165 +++++++++--------- 1 file changed, 82 insertions(+), 83 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 43857933..92ddb4bf 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -47,6 +47,8 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex @@ -215,9 +217,13 @@ class DepotDownloader @JvmOverloads constructor( processItems() } - // Launch the file writing loop - scope.launch { - processFileWrites() + // Launch the file writing receivers + List(maxFileWrites) { + scope.launch { + fileWriteChannel.receiveAsFlow().collect { item -> + processFileWrites(item) + } + } } } @@ -1702,6 +1708,7 @@ class DepotDownloader @JvmOverloads constructor( override fun close() { processingChannel.close() + fileWriteChannel.close() scope.cancel("DepotDownloader Closing") @@ -1718,97 +1725,89 @@ class DepotDownloader @JvmOverloads constructor( logger = null } - private suspend fun processFileWrites() = coroutineScope { - while (!completionFuture.isDone && !completionFuture.isCancelled && !completionFuture.isCompletedExceptionally) { - List(maxFileWrites) { - async { - for (item in fileWriteChannel) { - val depot = item.depot - val depotDownloadCounter = item.depotDownloadCounter - val downloadCounter = item.downloadCounter - val written = item.written - val file = item.file - val fileStreamData = item.fileStreamData - val chunk = item.chunk - val chunkBuffer = item.chunkBuffer + private suspend fun processFileWrites(item: FileWriteItem) { + val depot = item.depot + val depotDownloadCounter = item.depotDownloadCounter + val downloadCounter = item.downloadCounter + val written = item.written + val file = item.file + val fileStreamData = item.fileStreamData + val chunk = item.chunk + val chunkBuffer = item.chunkBuffer - try { - fileStreamData.fileLock.lock() + try { + fileStreamData.fileLock.lock() - if (fileStreamData.fileHandle == null) { - val fileFinalPath = depot.installDir / file.fileName - fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) - } + if (fileStreamData.fileHandle == null) { + val fileFinalPath = depot.installDir / file.fileName + fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) + } - fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) - } finally { - fileStreamData.fileLock.unlock() - } + fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) + } finally { + fileStreamData.fileLock.unlock() + } - val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() - if (remainingChunks == 0) { - fileStreamData.fileHandle?.close() + val remainingChunks = fileStreamData.chunksToDownload.decrementAndGet() + if (remainingChunks == 0) { + fileStreamData.fileHandle?.close() - // File completed - notify with percentage - val sizeDownloaded = synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += written.toLong() - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - depotDownloadCounter.sizeDownloaded - } + // File completed - notify with percentage + val sizeDownloaded = synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + depotDownloadCounter.sizeDownloaded + } - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } - val fileFinalPath = depot.installDir / file.fileName - val depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) + val fileFinalPath = depot.installDir / file.fileName + val depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) - notifyListeners { listener -> - listener.onFileCompleted( - depotId = depot.depotId, - fileName = fileFinalPath.toString(), - depotPercentComplete = depotPercentage - ) - } + notifyListeners { listener -> + listener.onFileCompleted( + depotId = depot.depotId, + fileName = fileFinalPath.toString(), + depotPercentComplete = depotPercentage + ) + } - logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) - } else { - // Update counters and notify on chunk completion - val sizeDownloaded: Long - val depotPercentage: Float - val compressedBytes: Long - val uncompressedBytes: Long - - synchronized(depotDownloadCounter) { - depotDownloadCounter.sizeDownloaded += written.toLong() - depotDownloadCounter.depotBytesCompressed += chunk.compressedLength - depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength - - sizeDownloaded = depotDownloadCounter.sizeDownloaded - compressedBytes = depotDownloadCounter.depotBytesCompressed - uncompressedBytes = depotDownloadCounter.depotBytesUncompressed - depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) - } + logger?.debug("%.2f%% %s".format(depotPercentage, fileFinalPath)) + } else { + // Update counters and notify on chunk completion + val sizeDownloaded: Long + val depotPercentage: Float + val compressedBytes: Long + val uncompressedBytes: Long - synchronized(downloadCounter) { - downloadCounter.totalBytesCompressed += chunk.compressedLength - downloadCounter.totalBytesUncompressed += chunk.uncompressedLength - } + synchronized(depotDownloadCounter) { + depotDownloadCounter.sizeDownloaded += written.toLong() + depotDownloadCounter.depotBytesCompressed += chunk.compressedLength + depotDownloadCounter.depotBytesUncompressed += chunk.uncompressedLength + + sizeDownloaded = depotDownloadCounter.sizeDownloaded + compressedBytes = depotDownloadCounter.depotBytesCompressed + uncompressedBytes = depotDownloadCounter.depotBytesUncompressed + depotPercentage = (sizeDownloaded.toFloat() / depotDownloadCounter.completeDownloadSize) + } - notifyListeners { listener -> - listener.onChunkCompleted( - depotId = depot.depotId, - depotPercentComplete = depotPercentage, - compressedBytes = compressedBytes, - uncompressedBytes = uncompressedBytes - ) - } - } - } - } - }.awaitAll() + synchronized(downloadCounter) { + downloadCounter.totalBytesCompressed += chunk.compressedLength + downloadCounter.totalBytesUncompressed += chunk.uncompressedLength + } + + notifyListeners { listener -> + listener.onChunkCompleted( + depotId = depot.depotId, + depotPercentComplete = depotPercentage, + compressedBytes = compressedBytes, + uncompressedBytes = uncompressedBytes + ) + } } } } From 61375977f6ab76610ac941069a16aefcb0c634e7 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Mon, 15 Dec 2025 16:19:56 +0800 Subject: [PATCH 33/44] update notifyListeners to run on main thread instead launching other new thread --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 92ddb4bf..d7113023 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -1528,8 +1528,8 @@ class DepotDownloader @JvmOverloads constructor( listeners.remove(listener) } - private suspend fun notifyListeners(action: (IDownloadListener) -> Unit) { - coroutineScope { + private fun notifyListeners(action: (IDownloadListener) -> Unit) { + scope.launch(Dispatchers.Main) { listeners.forEach { listener -> action(listener) } } } From 7f66ba248e4efeab346de10083f46f461e0cf4ec Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Mon, 15 Dec 2025 23:01:26 +0800 Subject: [PATCH 34/44] Revert "update notifyListeners to run on main thread instead launching other new thread" This reverts commit 110abe939ed2c7afba22d9636c99295815358d39. --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index d7113023..92ddb4bf 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -1528,8 +1528,8 @@ class DepotDownloader @JvmOverloads constructor( listeners.remove(listener) } - private fun notifyListeners(action: (IDownloadListener) -> Unit) { - scope.launch(Dispatchers.Main) { + private suspend fun notifyListeners(action: (IDownloadListener) -> Unit) { + coroutineScope { listeners.forEach { listener -> action(listener) } } } From 66fe3cfd8e88de6fb2b1f1a97f27d03b3746caa1 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Tue, 16 Dec 2025 11:18:16 +0800 Subject: [PATCH 35/44] Improves concurrent depot downloading Refactors depot downloading to use separate coroutines for network chunk processing and file writing, improving concurrency and overall download speed. This change introduces dedicated coroutine workers for network chunk downloads and file writing, managing these tasks in parallel. It also ensures proper cleanup and channel closing, preventing resource leaks and ensuring all workers complete their tasks before the download is considered finished. Additionally, it improves the handling of last depot scenarios. --- .../depotdownloader/DepotDownloader.kt | 199 +++++++++++------- 1 file changed, 126 insertions(+), 73 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 92ddb4bf..7fcc501b 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -40,6 +40,8 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -49,6 +51,7 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive import kotlinx.coroutines.flow.consumeAsFlow import kotlinx.coroutines.flow.receiveAsFlow +import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex @@ -95,6 +98,7 @@ import kotlin.text.toLongOrNull * @param debug Enables detailed logging of all operations via [LogManager] * @param useLanCache Attempts to detect and use local Steam cache servers (e.g., LANCache) for faster downloads on local networks * @param maxDownloads Number of concurrent chunk downloads. Automatically increased to 25 when a LAN cache is detected. Default: 8 + * @param maxFileWrites Number of concurrent files being written. Default: 8 * @param androidEmulation Forces "Windows" as the default OS filter. Used when running Android games in PC emulators that expect Windows builds. * * @author Oxters @@ -153,13 +157,22 @@ class DepotDownloader @JvmOverloads constructor( private var processingChannel = Channel(Channel.UNLIMITED) + private var networkChunkQueue = Channel(Channel.UNLIMITED) + private var fileWriteChannel = Channel(Channel.UNLIMITED) private var steam3: Steam3Session? = null + private var processingItemsMap = mutableMapOf() + + private var networkWorkers: List = emptyList() + private var fileWriteWorkers: List = emptyList() + // region [REGION] Private data classes. private data class NetworkChunkItem( + val downloadCounter: GlobalDownloadCounter, + val depotFilesData: DepotFilesData, val fileStreamData: FileStreamData, val fileData: FileData, val chunk: ChunkData, @@ -216,15 +229,6 @@ class DepotDownloader @JvmOverloads constructor( scope.launch { processItems() } - - // Launch the file writing receivers - List(maxFileWrites) { - scope.launch { - fileWriteChannel.receiveAsFlow().collect { item -> - processFileWrites(item) - } - } - } } // region [REGION] Downloading Operations @@ -507,7 +511,7 @@ class DepotDownloader @JvmOverloads constructor( } } - downloadSteam3(infos) + downloadSteam3(appId, infos) } @Throws(IllegalStateException::class) @@ -833,7 +837,7 @@ class DepotDownloader @JvmOverloads constructor( return info != null && info["FreeToDownload"].asBoolean() } - private suspend fun downloadSteam3(depots: List): Unit = coroutineScope { + private suspend fun downloadSteam3(mainAppId: Int, depots: List): Unit = coroutineScope { val maxNumServers = maxDownloads.coerceIn(20, 64) // Hard clamp at 64. Not sure how high we can go. cdnClientPool?.updateServerList(maxNumServers) @@ -864,8 +868,18 @@ class DepotDownloader @JvmOverloads constructor( } } - depotsToDownload.forEach { depotFileData -> - downloadSteam3DepotFiles(downloadCounter, depotFileData, allFileNamesAllDepots) + if (depotsToDownload.isEmpty()) { + finishDepotDownload(mainAppId) + } else { + depotsToDownload.forEachIndexed { index, depotFileData -> + downloadSteam3DepotFiles( + mainAppId, + downloadCounter, + depotFileData, + allFileNamesAllDepots, + index == depotsToDownload.size - 1 + ) + } } logger?.debug( @@ -1075,9 +1089,11 @@ class DepotDownloader @JvmOverloads constructor( @OptIn(DelicateCoroutinesApi::class) private suspend fun downloadSteam3DepotFiles( + mainAppId: Int, downloadCounter: GlobalDownloadCounter, depotFilesData: DepotFilesData, allFileNamesAllDepots: HashSet, + isLastDepot: Boolean, ) = withContext(Dispatchers.IO) { val depot = depotFilesData.depotDownloadInfo val depotCounter = depotFilesData.depotCounter @@ -1085,11 +1101,10 @@ class DepotDownloader @JvmOverloads constructor( logger?.debug("Downloading depot ${depot.depotId}") val files = depotFilesData.filteredFiles.filter { !it.flags.contains(EDepotFileFlag.Directory) } - val networkChunkQueue = Channel(Channel.UNLIMITED) try { coroutineScope { - // First parallel loop - process files and enqueue chunks + // Second parallel loop - process files and enqueue chunks files.chunked(50).forEach { batch -> yield() @@ -1099,33 +1114,33 @@ class DepotDownloader @JvmOverloads constructor( downloadCounter = downloadCounter, depotFilesData = depotFilesData, file = file, - networkChunkQueue = networkChunkQueue ) } }.awaitAll() } // Close the channel to signal no more items will be added - networkChunkQueue.close() + if (isLastDepot) { + networkChunkQueue.close() - // Second parallel loop - process chunks from queue - List(maxDownloads) { - async { - for (item in networkChunkQueue) { - downloadSteam3DepotFileChunk( - downloadCounter = downloadCounter, - depotFilesData = depotFilesData, - file = item.fileData, - fileStreamData = item.fileStreamData, - chunk = item.chunk - ) - } - } - }.awaitAll() + // Wait for all network workers to finish processing remaining items + networkWorkers.joinAll() + } } } finally { - if (!networkChunkQueue.isClosedForSend) { - networkChunkQueue.close() + if (isLastDepot) { + if (!networkChunkQueue.isClosedForSend) { + // Close the channel to signal no more items will be added + networkChunkQueue.close() + } + + // Close file write channel since no more network workers can send to it + if (!fileWriteChannel.isClosedForSend) { + fileWriteChannel.close() + } + + // Wait for all file write workers to finish processing remaining items + fileWriteWorkers.joinAll() } } @@ -1170,13 +1185,16 @@ class DepotDownloader @JvmOverloads constructor( } logger?.debug("Depot ${depot.depotId} - Downloaded ${depotCounter.depotBytesCompressed} bytes (${depotCounter.depotBytesUncompressed} bytes uncompressed)") + + if (isLastDepot) { + finishDepotDownload(mainAppId) + } } private suspend fun downloadSteam3DepotFile( downloadCounter: GlobalDownloadCounter, depotFilesData: DepotFilesData, file: FileData, - networkChunkQueue: Channel, ) = withContext(Dispatchers.IO) { ensureActive() @@ -1378,10 +1396,12 @@ class DepotDownloader @JvmOverloads constructor( neededChunks!!.forEach { chunk -> networkChunkQueue.send( NetworkChunkItem( + downloadCounter = downloadCounter, + depotFilesData = depotFilesData, fileStreamData = fileStreamData, fileData = file, chunk = chunk, - totalChunksForFile = neededChunks!!.size + totalChunksForFile = neededChunks!!.size, ) ) } @@ -1516,6 +1536,15 @@ class DepotDownloader @JvmOverloads constructor( return false } + private suspend fun finishDepotDownload(mainAppId: Int) { + val appItem = processingItemsMap[mainAppId] + if (appItem != null) { + notifyListeners { it.onDownloadCompleted(appItem) } + } + + completionFuture.complete(null) + } + // endregion // region [REGION] Listener Operations @@ -1595,6 +1624,30 @@ class DepotDownloader @JvmOverloads constructor( } } + // First pre-launch coroutines + networkWorkers = List(maxDownloads) { + scope.launch { + networkChunkQueue.receiveAsFlow().collect { item -> + downloadSteam3DepotFileChunk( + downloadCounter = item.downloadCounter, + depotFilesData = item.depotFilesData, + file = item.fileData, + fileStreamData = item.fileStreamData, + chunk = item.chunk, + ) + } + } + } + + // Launch the file writing receivers + fileWriteWorkers = List(maxFileWrites) { + scope.launch { + fileWriteChannel.receiveAsFlow().collect { item -> + processFileWrites(item) + } + } + } + for (item in processingChannel) { try { ensureActive() @@ -1606,6 +1659,8 @@ class DepotDownloader @JvmOverloads constructor( installToGameNameDirectory = item.installToGameNameDirectory, ) + processingItemsMap[item.appId] = item + when (item) { is PubFileItem -> { logger?.debug("Downloading PUB File for ${item.appId}") @@ -1680,52 +1735,17 @@ class DepotDownloader @JvmOverloads constructor( ) } } - - notifyListeners { it.onDownloadCompleted(item) } } catch (e: Exception) { logger?.error("Error downloading item ${item.appId}: ${e.message}", e) notifyListeners { it.onDownloadFailed(item, e) } } } - - completionFuture.complete(null) - } - - /** - * Returns a CompletableFuture that completes when all queued downloads finish. - * @return CompletableFuture that completes when all downloads finish - */ - fun getCompletion(): CompletableFuture = completionFuture - - /** - * Blocks the current thread until all queued downloads complete. - * Convenience method that calls `getCompletion().join()`. - * @throws CompletionException if any download fails - */ - fun awaitCompletion() { - completionFuture.join() } - override fun close() { - processingChannel.close() - fileWriteChannel.close() - - scope.cancel("DepotDownloader Closing") - - httpClient.close() - - listeners.clear() - - steam3?.close() - steam3 = null - - cdnClientPool?.close() - cdnClientPool = null - - logger = null - } + private suspend fun processFileWrites(item: FileWriteItem): Unit = withContext(Dispatchers.IO) { + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() - private suspend fun processFileWrites(item: FileWriteItem) { val depot = item.depot val depotDownloadCounter = item.depotDownloadCounter val downloadCounter = item.downloadCounter @@ -1810,4 +1830,37 @@ class DepotDownloader @JvmOverloads constructor( } } } + + /** + * Returns a CompletableFuture that completes when all queued downloads finish. + * @return CompletableFuture that completes when all downloads finish + */ + fun getCompletion(): CompletableFuture = completionFuture + + /** + * Blocks the current thread until all queued downloads complete. + * Convenience method that calls `getCompletion().join()`. + * @throws CompletionException if any download fails + */ + fun awaitCompletion() { + completionFuture.join() + } + + override fun close() { + processingChannel.close() + + scope.cancel("DepotDownloader Closing") + + httpClient.close() + + listeners.clear() + + steam3?.close() + steam3 = null + + cdnClientPool?.close() + cdnClientPool = null + + logger = null + } } From d9d00710446d781aba0ab6df3a5d3b35bed8be70 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 15 Dec 2025 21:48:26 -0600 Subject: [PATCH 36/44] Lint formatting --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 7fcc501b..3dbf71e3 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -40,7 +40,6 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async @@ -49,7 +48,6 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive -import kotlinx.coroutines.flow.consumeAsFlow import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch From 5cdb7749985d717c8941d2848a451fe1c8471378 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Thu, 18 Dec 2025 01:11:34 +0800 Subject: [PATCH 37/44] Refactors chunk processing with Flows Migrates from Channels to Flows for chunk processing to improve concurrency and simplify data flow management. This change introduces dedicated flows for decompression and file writing, enabling better control over resource utilization and error handling. It also decouples network I/O from decompression and writing operations by processing in parallel, preventing potential bottlenecks. Increases chunk processing throughput and reduces latency by better utilizing available CPU cores. --- .../depotdownloader/DepotDownloader.kt | 209 +++++++++++------- .../dragonbra/javasteam/steam/cdn/Client.kt | 27 ++- 2 files changed, 151 insertions(+), 85 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 3dbf71e3..fbe15ecd 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -15,6 +15,7 @@ import `in`.dragonbra.javasteam.enums.EAppInfoSection import `in`.dragonbra.javasteam.enums.EDepotFileFlag import `in`.dragonbra.javasteam.enums.EWorkshopFileType import `in`.dragonbra.javasteam.steam.cdn.ClientLancache +import `in`.dragonbra.javasteam.steam.cdn.DepotChunk import `in`.dragonbra.javasteam.steam.cdn.Server import `in`.dragonbra.javasteam.steam.handlers.steamapps.License import `in`.dragonbra.javasteam.steam.handlers.steamapps.callback.LicenseListCallback @@ -45,11 +46,11 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.cancel +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive -import kotlinx.coroutines.flow.receiveAsFlow -import kotlinx.coroutines.joinAll +import kotlinx.coroutines.flow.* import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex @@ -96,13 +97,15 @@ import kotlin.text.toLongOrNull * @param debug Enables detailed logging of all operations via [LogManager] * @param useLanCache Attempts to detect and use local Steam cache servers (e.g., LANCache) for faster downloads on local networks * @param maxDownloads Number of concurrent chunk downloads. Automatically increased to 25 when a LAN cache is detected. Default: 8 - * @param maxFileWrites Number of concurrent files being written. Default: 8 + * @param maxDecompress Number of concurrent chunk decompress. Default: 8 + * @param maxFileWrites Number of concurrent files being written. Default: 1 * @param androidEmulation Forces "Windows" as the default OS filter. Used when running Android games in PC emulators that expect Windows builds. * * @author Oxters * @author Lossy * @since Oct 29, 2024 */ +@OptIn(ExperimentalCoroutinesApi::class) @Suppress("unused") class DepotDownloader @JvmOverloads constructor( private val steamClient: SteamClient, @@ -110,7 +113,8 @@ class DepotDownloader @JvmOverloads constructor( private val debug: Boolean = false, private val useLanCache: Boolean = false, private var maxDownloads: Int = 8, - private var maxFileWrites: Int = 8, + private var maxDecompress: Int = 8, + private var maxFileWrites: Int = 1, private val androidEmulation: Boolean = false, ) : Closeable { @@ -155,17 +159,16 @@ class DepotDownloader @JvmOverloads constructor( private var processingChannel = Channel(Channel.UNLIMITED) - private var networkChunkQueue = Channel(Channel.UNLIMITED) + private val networkChunkFlow = MutableSharedFlow(extraBufferCapacity = Int.MAX_VALUE) - private var fileWriteChannel = Channel(Channel.UNLIMITED) + private val pendingChunks = AtomicInteger(0) + + private var chunkProcessingJob: Job? = null private var steam3: Steam3Session? = null private var processingItemsMap = mutableMapOf() - private var networkWorkers: List = emptyList() - private var fileWriteWorkers: List = emptyList() - // region [REGION] Private data classes. private data class NetworkChunkItem( @@ -177,17 +180,28 @@ class DepotDownloader @JvmOverloads constructor( val totalChunksForFile: Int, ) - private data class FileWriteItem( + private data class DecompressItem( val depot: DepotDownloadInfo, val depotDownloadCounter: DepotDownloadCounter, val downloadCounter: GlobalDownloadCounter, - val written: Int, + val downloaded: Int, val file: FileData, val fileStreamData: FileStreamData, val chunk: ChunkData, val chunkBuffer: ByteArray, ) + private data class FileWriteItem( + val depot: DepotDownloadInfo, + val depotDownloadCounter: DepotDownloadCounter, + val downloadCounter: GlobalDownloadCounter, + val file: FileData, + val fileStreamData: FileStreamData, + val chunk: ChunkData, + val decompressed: Int, + val decompressedBuffer: ByteArray, + ) + private data class DirectoryResult(val success: Boolean, val installDir: Path?) private data class Config( @@ -229,6 +243,50 @@ class DepotDownloader @JvmOverloads constructor( } } + private fun createChunkProcessingFlow(): kotlinx.coroutines.flow.Flow = networkChunkFlow + .flatMapMerge(concurrency = maxDownloads) { item -> + flow { + try { + val result = downloadSteam3DepotFileChunk( + downloadCounter = item.downloadCounter, + depotFilesData = item.depotFilesData, + file = item.fileData, + fileStreamData = item.fileStreamData, + chunk = item.chunk + ) + if (result != null) { + emit(result) + } + } catch (e: Exception) { + logger?.error("Error downloading chunk: ${e.message}", e) + } + }.flowOn(Dispatchers.IO) + } + .flatMapMerge(concurrency = maxDecompress) { item -> + flow { + try { + val result = processFileDecompress(item) + if (result != null) { + emit(result) + } + } catch (e: Exception) { + logger?.error("Error decompressing chunk: ${e.message}", e) + } + }.flowOn(Dispatchers.Default) + } + .flatMapMerge(concurrency = maxFileWrites) { item -> + flow { + try { + processFileWrites(item) + pendingChunks.decrementAndGet() + emit(Unit) + } catch (e: Exception) { + logger?.error("Error writing file: ${e.message}", e) + pendingChunks.decrementAndGet() + } + }.flowOn(Dispatchers.IO) + } + // region [REGION] Downloading Operations private suspend fun processPublishedFile( @@ -1116,29 +1174,22 @@ class DepotDownloader @JvmOverloads constructor( } }.awaitAll() } - - // Close the channel to signal no more items will be added - if (isLastDepot) { - networkChunkQueue.close() - - // Wait for all network workers to finish processing remaining items - networkWorkers.joinAll() - } } } finally { if (isLastDepot) { - if (!networkChunkQueue.isClosedForSend) { - // Close the channel to signal no more items will be added - networkChunkQueue.close() - } + logger?.debug("Waiting for ${pendingChunks.get()} pending chunks to complete for depot ${depot.depotId}") - // Close file write channel since no more network workers can send to it - if (!fileWriteChannel.isClosedForSend) { - fileWriteChannel.close() + // Wait for all pending chunks to complete processing + while (pendingChunks.get() > 0) { + kotlinx.coroutines.delay(100) } - // Wait for all file write workers to finish processing remaining items - fileWriteWorkers.joinAll() + logger?.debug("All chunks completed, canceling processing job for depot ${depot.depotId}") + + // Cancel the continuous flow job since no more chunks will be added + chunkProcessingJob?.cancel() + + logger?.debug("Canceled chunk processing job for depot ${depot.depotId}") } } @@ -1392,7 +1443,8 @@ class DepotDownloader @JvmOverloads constructor( ) neededChunks!!.forEach { chunk -> - networkChunkQueue.send( + pendingChunks.incrementAndGet() + networkChunkFlow.tryEmit( NetworkChunkItem( downloadCounter = downloadCounter, depotFilesData = depotFilesData, @@ -1411,7 +1463,7 @@ class DepotDownloader @JvmOverloads constructor( file: FileData, fileStreamData: FileStreamData, chunk: ChunkData, - ): Unit = withContext(Dispatchers.IO) { + ): DecompressItem? = withContext(Dispatchers.Default) { ensureActive() val depot = depotFilesData.depotDownloadInfo @@ -1419,8 +1471,8 @@ class DepotDownloader @JvmOverloads constructor( val chunkID = Strings.toHex(chunk.chunkID) - var written = 0 - val chunkBuffer = ByteArray(chunk.uncompressedLength) + var downloaded = 0 + val chunkBuffer = ByteArray(chunk.compressedLength) do { ensureActive() @@ -1445,7 +1497,7 @@ class DepotDownloader @JvmOverloads constructor( logger?.debug("Downloading chunk $chunkID from $connection with ${cdnClientPool!!.proxyServer ?: "no proxy"}") - written = cdnClientPool!!.cdnClient!!.downloadDepotChunk( + downloaded = cdnClientPool!!.cdnClient!!.downloadDepotChunk( depotId = depot.depotId, chunk = chunk, server = connection, @@ -1489,9 +1541,9 @@ class DepotDownloader @JvmOverloads constructor( cdnClientPool!!.returnBrokenConnection(connection) logger?.error("Encountered unexpected error downloading chunk $chunkID", e) } - } while (written == 0) + } while (downloaded == 0) - if (written == 0) { + if (downloaded == 0) { logger?.error("Failed to find any server with chunk ${chunk.chunkID} for depot ${depot.depotId}. Aborting.") cancel() } @@ -1499,18 +1551,16 @@ class DepotDownloader @JvmOverloads constructor( // Throw the cancellation exception if requested so that this task is marked failed ensureActive() - // Queue the file write operation - fileWriteChannel.send( - FileWriteItem( - depot = depot, - depotDownloadCounter = depotDownloadCounter, - downloadCounter = downloadCounter, - written = written, - file = file, - fileStreamData = fileStreamData, - chunk = chunk, - chunkBuffer = chunkBuffer, - ) + // Return the decompress item for the next stage in the pipeline + return@withContext DecompressItem( + depot = depot, + depotDownloadCounter = depotDownloadCounter, + downloadCounter = downloadCounter, + downloaded = downloaded, + file = file, + fileStreamData = fileStreamData, + chunk = chunk, + chunkBuffer = chunkBuffer, ) } @@ -1555,8 +1605,8 @@ class DepotDownloader @JvmOverloads constructor( listeners.remove(listener) } - private suspend fun notifyListeners(action: (IDownloadListener) -> Unit) { - coroutineScope { + private fun notifyListeners(action: (IDownloadListener) -> Unit) { + scope.launch(Dispatchers.IO) { listeners.forEach { listener -> action(listener) } } } @@ -1622,28 +1672,9 @@ class DepotDownloader @JvmOverloads constructor( } } - // First pre-launch coroutines - networkWorkers = List(maxDownloads) { - scope.launch { - networkChunkQueue.receiveAsFlow().collect { item -> - downloadSteam3DepotFileChunk( - downloadCounter = item.downloadCounter, - depotFilesData = item.depotFilesData, - file = item.fileData, - fileStreamData = item.fileStreamData, - chunk = item.chunk, - ) - } - } - } - - // Launch the file writing receivers - fileWriteWorkers = List(maxFileWrites) { - scope.launch { - fileWriteChannel.receiveAsFlow().collect { item -> - processFileWrites(item) - } - } + // Launch the chunk processing pipeline using Flow + chunkProcessingJob = scope.launch { + createChunkProcessingFlow().collect() } for (item in processingChannel) { @@ -1740,6 +1771,36 @@ class DepotDownloader @JvmOverloads constructor( } } + private suspend fun processFileDecompress(item: DecompressItem): FileWriteItem? = withContext(Dispatchers.Default) { + // Throw the cancellation exception if requested so that this task is marked failed + ensureActive() + + val depot = item.depot + val depotKey = depot.depotKey + val downloaded = item.downloaded + val chunk = item.chunk + val chunkBuffer = item.chunkBuffer + + var written = downloaded + var decompressedBuffer = chunkBuffer + + if (depotKey != null) { + decompressedBuffer = ByteArray(chunk.uncompressedLength) + written = DepotChunk.process(chunk, chunkBuffer, decompressedBuffer, depotKey) + } + + return@withContext FileWriteItem( + depot = depot, + depotDownloadCounter = item.depotDownloadCounter, + downloadCounter = item.downloadCounter, + file = item.file, + fileStreamData = item.fileStreamData, + chunk = chunk, + decompressed = written, + decompressedBuffer = decompressedBuffer, + ) + } + private suspend fun processFileWrites(item: FileWriteItem): Unit = withContext(Dispatchers.IO) { // Throw the cancellation exception if requested so that this task is marked failed ensureActive() @@ -1747,11 +1808,11 @@ class DepotDownloader @JvmOverloads constructor( val depot = item.depot val depotDownloadCounter = item.depotDownloadCounter val downloadCounter = item.downloadCounter - val written = item.written val file = item.file val fileStreamData = item.fileStreamData val chunk = item.chunk - val chunkBuffer = item.chunkBuffer + val written = item.decompressed + val decompressedBuffer = item.decompressedBuffer try { fileStreamData.fileLock.lock() @@ -1761,7 +1822,7 @@ class DepotDownloader @JvmOverloads constructor( fileStreamData.fileHandle = filesystem.openReadWrite(fileFinalPath) } - fileStreamData.fileHandle!!.write(chunk.offset, chunkBuffer, 0, written) + fileStreamData.fileHandle!!.write(chunk.offset, decompressedBuffer, 0, written) } finally { fileStreamData.fileLock.unlock() } diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index c34da41c..25db82f2 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -207,8 +207,8 @@ class Client(steamClient: SteamClient) : Closeable { throw IllegalArgumentException("The destination buffer must be longer than the chunk CompressedLength (since no depot key was provided).") } } else { - if (destination.size < chunk.uncompressedLength) { - throw IllegalArgumentException("The destination buffer must be longer than the chunk UncompressedLength.") + if (destination.size != chunk.compressedLength) { + throw IllegalArgumentException("The destination buffer must be the same size as the chunk UncompressedLength.") } } @@ -267,12 +267,8 @@ class Client(steamClient: SteamClient) : Closeable { ) } - if (depotKey == null) { - System.arraycopy(responseBody, 0, destination, 0, contentLength) - return@withContext contentLength - } - - return@withContext DepotChunk.process(chunk, responseBody, destination, depotKey) + System.arraycopy(responseBody, 0, destination, 0, contentLength) + return@withContext contentLength } } catch (e: Exception) { logger.error("Failed to download a depot chunk ${request.url}: ${e.message}", e) @@ -364,16 +360,25 @@ class Client(steamClient: SteamClient) : Closeable { scope.launch { try { - val bytesWritten = downloadDepotChunk( + val downloadedBytes = ByteArray(chunk.compressedLength) + + val bytesDownloaded = downloadDepotChunk( depotId = depotId, chunk = chunk, server = server, - destination = destination, + destination = downloadedBytes, depotKey = depotKey, proxyServer = proxyServer, cdnAuthToken = cdnAuthToken ) - future.complete(bytesWritten) + + if (depotKey != null) { + val bytesProcessed = DepotChunk.process(chunk, downloadedBytes, destination, depotKey) + future.complete(bytesProcessed) + } else { + System.arraycopy(downloadedBytes, 0, destination, 0, bytesDownloaded) + future.complete(bytesDownloaded) + } } catch (e: Exception) { future.completeExceptionally(e) } From 5455bb2d91b2ec212eb85a9ceaf460713c3dc7f1 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Thu, 18 Dec 2025 02:06:13 +0800 Subject: [PATCH 38/44] Updates exception handling in CDN client Refines exception handling for chunk downloads in the CDN client. Adjusts the logic to validate destination buffer size based on whether a depot key is provided. --- .../dragonbra/javasteam/steam/cdn/Client.kt | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt index 25db82f2..5584132d 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/Client.kt @@ -202,14 +202,8 @@ class Client(steamClient: SteamClient) : Closeable { ): Int = withContext(Dispatchers.IO) { require(chunk.chunkID != null) { "Chunk must have a ChunkID." } - if (depotKey == null) { - if (destination.size < chunk.compressedLength) { - throw IllegalArgumentException("The destination buffer must be longer than the chunk CompressedLength (since no depot key was provided).") - } - } else { - if (destination.size != chunk.compressedLength) { - throw IllegalArgumentException("The destination buffer must be the same size as the chunk UncompressedLength.") - } + if (destination.size != chunk.compressedLength) { + throw IllegalArgumentException("The destination buffer must be the same size as the chunk CompressedLength (Since we take out decompression step from download") } val chunkID = Strings.toHex(chunk.chunkID) @@ -360,6 +354,16 @@ class Client(steamClient: SteamClient) : Closeable { scope.launch { try { + if (depotKey != null) { + if (destination.size < chunk.compressedLength) { + throw IllegalArgumentException("The destination buffer must be longer than the chunk CompressedLength (since no depot key was provided).") + } + } else { + if (destination.size != chunk.compressedLength) { + throw IllegalArgumentException("The destination buffer must be the same size as the chunk UncompressedLength.") + } + } + val downloadedBytes = ByteArray(chunk.compressedLength) val bytesDownloaded = downloadDepotChunk( From 61257af48dc9b728e04389f60b52e680bf889dc9 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Thu, 18 Dec 2025 02:12:31 +0800 Subject: [PATCH 39/44] Lint fix: imports specific Kotlin flow Imports the necessary Kotlin flow operators for the 'DepotDownloader' class to resolve compilation errors and ensure proper data stream processing. --- .../dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index fbe15ecd..e311b961 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -50,7 +50,11 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive -import kotlinx.coroutines.flow.* +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.flatMapMerge +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex From 571755b5225356311599642943aa17eed977a1c9 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 17 Dec 2025 12:58:04 -0600 Subject: [PATCH 40/44] Fix lexicographic order --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index e311b961..e6b4f83a 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -41,12 +41,12 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.DelicateCoroutinesApi import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.cancel -import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.ensureActive From 3cbf9db0420901787a71eb9d9993bf5aaa65c714 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Fri, 19 Dec 2025 11:36:16 +0800 Subject: [PATCH 41/44] Changes dispatcher for download task Changes the dispatcher used for the download task to IO. This ensures that download tasks, which are I/O bound, run on a dispatcher that is optimized for I/O operations, preventing potential blocking of the default dispatcher. --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index e6b4f83a..6f7923d5 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -1467,7 +1467,7 @@ class DepotDownloader @JvmOverloads constructor( file: FileData, fileStreamData: FileStreamData, chunk: ChunkData, - ): DecompressItem? = withContext(Dispatchers.Default) { + ): DecompressItem? = withContext(Dispatchers.IO) { ensureActive() val depot = depotFilesData.depotDownloadInfo From 0cd6d13ca2b4ba9e4b57d64e26ad3ef704a1d6c0 Mon Sep 17 00:00:00 2001 From: Joshua Tam <297250+joshuatam@users.noreply.github.com> Date: Fri, 19 Dec 2025 12:58:35 +0800 Subject: [PATCH 42/44] Adds parent job cancellation support Ensures that the downloader is cancelled when its parent job is cancelled, allowing for better control and resource management in complex workflows. --- .../in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index 6f7923d5..e6ff5a63 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -104,6 +104,7 @@ import kotlin.text.toLongOrNull * @param maxDecompress Number of concurrent chunk decompress. Default: 8 * @param maxFileWrites Number of concurrent files being written. Default: 1 * @param androidEmulation Forces "Windows" as the default OS filter. Used when running Android games in PC emulators that expect Windows builds. + * @param parentJob Parent job for the downloader. If provided, the downloader will be cancelled when the parent job is cancelled. * * @author Oxters * @author Lossy @@ -120,6 +121,7 @@ class DepotDownloader @JvmOverloads constructor( private var maxDecompress: Int = 8, private var maxFileWrites: Int = 1, private val androidEmulation: Boolean = false, + private val parentJob: Job? = null, ) : Closeable { companion object { @@ -153,7 +155,7 @@ class DepotDownloader @JvmOverloads constructor( private val progressUpdateInterval = 500L // ms - private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO + SupervisorJob(parentJob)) private var cdnClientPool: CDNClientPool? = null From d5db49f46569e29c1190fbd78459920e58838901 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Mon, 22 Dec 2025 21:55:46 -0600 Subject: [PATCH 43/44] Lint cleanup --- .../depotdownloader/DepotDownloader.kt | 18 ++++++++++-------- .../javasteam/steam/cdn/ClientLancache.kt | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt index e6ff5a63..539717d0 100644 --- a/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt +++ b/javasteam-depotdownloader/src/main/kotlin/in/dragonbra/javasteam/depotdownloader/DepotDownloader.kt @@ -186,6 +186,7 @@ class DepotDownloader @JvmOverloads constructor( val totalChunksForFile: Int, ) + @Suppress("ArrayInDataClass") private data class DecompressItem( val depot: DepotDownloadInfo, val depotDownloadCounter: DepotDownloadCounter, @@ -197,6 +198,7 @@ class DepotDownloader @JvmOverloads constructor( val chunkBuffer: ByteArray, ) + @Suppress("ArrayInDataClass") private data class FileWriteItem( val depot: DepotDownloadInfo, val depotDownloadCounter: DepotDownloadCounter, @@ -250,8 +252,8 @@ class DepotDownloader @JvmOverloads constructor( } private fun createChunkProcessingFlow(): kotlinx.coroutines.flow.Flow = networkChunkFlow - .flatMapMerge(concurrency = maxDownloads) { item -> - flow { + .flatMapMerge(concurrency = maxDownloads) { item -> + flow { try { val result = downloadSteam3DepotFileChunk( downloadCounter = item.downloadCounter, @@ -268,8 +270,8 @@ class DepotDownloader @JvmOverloads constructor( } }.flowOn(Dispatchers.IO) } - .flatMapMerge(concurrency = maxDecompress) { item -> - flow { + .flatMapMerge(concurrency = maxDecompress) { item -> + flow { try { val result = processFileDecompress(item) if (result != null) { @@ -280,8 +282,8 @@ class DepotDownloader @JvmOverloads constructor( } }.flowOn(Dispatchers.Default) } - .flatMapMerge(concurrency = maxFileWrites) { item -> - flow { + .flatMapMerge(concurrency = maxFileWrites) { item -> + flow { try { processFileWrites(item) pendingChunks.decrementAndGet() @@ -1469,7 +1471,7 @@ class DepotDownloader @JvmOverloads constructor( file: FileData, fileStreamData: FileStreamData, chunk: ChunkData, - ): DecompressItem? = withContext(Dispatchers.IO) { + ): DecompressItem = withContext(Dispatchers.IO) { ensureActive() val depot = depotFilesData.depotDownloadInfo @@ -1777,7 +1779,7 @@ class DepotDownloader @JvmOverloads constructor( } } - private suspend fun processFileDecompress(item: DecompressItem): FileWriteItem? = withContext(Dispatchers.Default) { + private suspend fun processFileDecompress(item: DecompressItem): FileWriteItem = withContext(Dispatchers.Default) { // Throw the cancellation exception if requested so that this task is marked failed ensureActive() diff --git a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt index 0eb9e699..442920b8 100644 --- a/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt +++ b/src/main/java/in/dragonbra/javasteam/steam/cdn/ClientLancache.kt @@ -66,6 +66,7 @@ object ClientLancache { return when (firstOctet) { 10 -> true + 172 -> { val secondOctet = bytes[1].toInt() and 0xFF secondOctet in 16..<32 From 86caf1785b6b2b4293556dbb5749a1dbbc46ccf0 Mon Sep 17 00:00:00 2001 From: LossyDragon Date: Wed, 24 Dec 2025 09:52:32 -0600 Subject: [PATCH 44/44] Prepare for release --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 84e27e9e..7c2a0dcd 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -20,7 +20,7 @@ plugins { allprojects { group = "in.dragonbra" - version = "1.8.0-SNAPSHOT" + version = "1.8.0" } repositories {