From 5655e1591d49ceca46f9bb60b903946c7a72e566 Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Sun, 23 Jul 2023 23:32:01 +0200 Subject: [PATCH 1/8] Quilt? --- Cargo.lock | 11 +++ Cargo.toml | 1 + helixlauncher-meta/src/util.rs | 18 +++++ src/intermediary.rs | 75 ++++++++++++++++++++ src/main.rs | 65 +++++++++++++++++ src/quilt.rs | 123 +++++++++++++++++++++++++++++++++ 6 files changed, 293 insertions(+) create mode 100644 src/intermediary.rs create mode 100644 src/quilt.rs diff --git a/Cargo.lock b/Cargo.lock index 15f6212..676d846 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -660,6 +660,7 @@ dependencies = [ "indexmap 2.0.0", "lazy_static", "maven-version-rs", + "quick-xml", "regex", "reqwest", "serde", @@ -1106,6 +1107,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.31" diff --git a/Cargo.toml b/Cargo.toml index 215ae8b..99f8ba1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ helixlauncher-meta = {path = "helixlauncher-meta"} indexmap = { version = "2.0.0", features = ["serde"] } lazy_static = "1.4.0" maven-version-rs = "0.1.0" +quick-xml = { version = "0.29.0", features = ["serde", "serialize"] } regex = "1.7.3" reqwest = {version = "0.11", features = ["json"]} serde = {version = "1", features = ["derive"]} diff --git a/helixlauncher-meta/src/util.rs b/helixlauncher-meta/src/util.rs index db29335..c463b34 100644 --- a/helixlauncher-meta/src/util.rs +++ b/helixlauncher-meta/src/util.rs @@ -79,6 +79,24 @@ impl Display for GradleSpecifier { } } +impl GradleSpecifier { + pub fn to_url(&self, base_repo: &str) -> String { + format!( + "{}{}/{}/{}/{}-{}{}.{}", + base_repo, + self.group.replace(".", "/"), + self.artifact, + self.version, + self.artifact, + self.version, + self.classifier + .as_ref() + .map_or("".to_string(), |it| "-".to_string() + &it), + self.extension + ) + } +} + cfg_if::cfg_if! { if #[cfg(windows)] { pub const CURRENT_OS: component::OsName = component::OsName::Windows; diff --git a/src/intermediary.rs b/src/intermediary.rs new file mode 100644 index 0000000..fadbd48 --- /dev/null +++ b/src/intermediary.rs @@ -0,0 +1,75 @@ +use std::{collections::BTreeSet, fs, path::Path, str::FromStr}; + +use anyhow::Result; +use chrono::Utc; +use helixlauncher_meta::{ + component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, + index::Index, util::GradleSpecifier, +}; +use reqwest::Client; + +use crate::{Metadata, get_hash, get_size}; + +pub async fn process(client: &Client) -> Result<()> { + let out_base = Path::new("out/net.fabricmc.intermediary"); + fs::create_dir_all(out_base)?; + + let mut index: Index = vec![]; + + for version in get_versions(client).await? { + let library = crate::Library { name: GradleSpecifier::from_str(&format!("net.fabricmc:intermediary:{version}")).unwrap(), url: "https://maven.fabricmc.net/".into() }; + let downloads = vec![Download { + name: library.name.clone(), + url: library.url.clone(), + hash: get_hash(client, &library).await?, + size: get_size(client, &library).await?.try_into().unwrap(), + }]; + let classpath = vec![ConditionalClasspathEntry::All(library.name)]; + + let component = Component { + format_version: 1, + assets: None, + conflicts: vec![], + id: "net.fabricmc.intermediary".into(), + jarmods: vec![], + natives: vec![], + release_time: Utc::now(), + version: version.clone(), + traits: BTreeSet::new(), + requires: vec![ComponentDependency { + id: "net.minecraft".into(), + version: Some(version), + }], + game_jar: None, + main_class: None, + game_arguments: vec![], + classpath, + downloads, + }; + + fs::write( + out_base.join(format!("{}.json", component.version)), + serde_json::to_string_pretty(&component)?, + )?; + + index.push(component.into()); + } + + index.sort_by(|x, y| y.release_time.cmp(&x.release_time)); + + fs::write( + out_base.join("index.json"), + serde_json::to_string_pretty(&index)?, + )?; + + Ok(()) +} + +async fn get_versions(client: &Client) -> Result> { + let response = client.get("https://maven.fabricmc.net/net/fabricmc/intermediary/maven-metadata.xml") + .header("User-Agent", "helixlauncher-meta (prototype)") + .send().await? + .text().await?; + let response: Metadata = quick_xml::de::from_str(&response)?; + Ok(response.versioning.versions.version) +} diff --git a/src/main.rs b/src/main.rs index 087c58d..a324a6f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,10 +5,17 @@ */ #![deny(rust_2018_idioms)] +use std::{fs, path::Path}; + use anyhow::Result; +use helixlauncher_meta::{component::{Component, Hash}, index::Index, util::GradleSpecifier}; +use reqwest::Client; +use serde::Deserialize; mod forge; +mod intermediary; mod mojang; +mod quilt; #[tokio::main] async fn main() -> Result<()> { @@ -18,5 +25,63 @@ async fn main() -> Result<()> { mojang::process()?; + // forge::process()?; + + quilt::process(&client).await?; + + intermediary::process(&client).await?; + Ok(()) } + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +#[allow(unused)] +pub(crate) struct Metadata { + artifact_id: String, + group_id: String, + versioning: Versioning, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +#[allow(unused)] +pub(crate) struct Versioning { + latest: String, + release: String, + versions: VersionList, + last_updated: String, +} + +#[derive(Deserialize)] +pub(crate) struct VersionList { + version: Vec, +} + +pub(crate) async fn get_hash(client: &Client, coord: &Library) -> Result { + Ok(Hash::SHA256( + client + .get(coord.name.to_url(&coord.url) + ".sha256") + .header("User-Agent", "helixlauncher-meta (prototype)") + .send() + .await? + .text() + .await?, + )) +} + +pub(crate) async fn get_size(client: &Client, coord: &Library) -> Result { + Ok(client + .head(coord.name.to_url(&coord.url)) + .header("User-Agent", "helixlauncher-meta (prototype)") + .send() + .await? + .content_length() + .expect("Cannot handle servers returning no content length")) +} + +#[derive(Deserialize, Debug)] +struct Library { + name: GradleSpecifier, + url: String, +} \ No newline at end of file diff --git a/src/quilt.rs b/src/quilt.rs new file mode 100644 index 0000000..a1354f1 --- /dev/null +++ b/src/quilt.rs @@ -0,0 +1,123 @@ +use std::{collections::BTreeSet, fs, path::Path}; + +use anyhow::{Context, Result}; +use chrono::{TimeZone, Utc}; +use helixlauncher_meta::{ + component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, + index::Index, +}; +use reqwest::Client; +use serde::Deserialize; + +use crate::{Metadata, Library}; +pub async fn process(client: &Client) -> Result<()> { + let out_base = Path::new("out/org.quiltmc.quilt-loader"); + fs::create_dir_all(out_base)?; + + let mut index: Index = vec![]; + + for loader_version in get_loader_versions(client).await? { + let response = client.get(format!("https://maven.quiltmc.org/repository/release/org/quiltmc/quilt-loader/{loader_version}/quilt-loader-{loader_version}.json")) + .header("User-Agent", "helixlauncher-meta (prototype)") + .send().await?; + + let release_time = Utc + .timestamp_millis_opt( + response + .headers() + .get("quilt-last-modified-timestamp") + .context("Error quilt did not provide release date in metadata")? + .to_str()? + .parse()?, + ) + .single() + .context("unable to parse release timestamp")?; + + let response: LoaderMeta = response.json().await?; + let mut downloads = vec![]; + let mut classpath = vec![]; + for library in response.libraries.common { + downloads.push(Download { + name: library.name.clone(), + url: library.url.clone(), + hash: crate::get_hash(client, &library).await?, + size: crate::get_size(client, &library).await?.try_into().unwrap(), + }); + classpath.push(ConditionalClasspathEntry::All(library.name)) + } + + let component = Component { + format_version: 1, + assets: None, + conflicts: vec![], + id: "org.quiltmc.quilt-loader".into(), + jarmods: vec![], + natives: vec![], + release_time, + version: loader_version, + traits: BTreeSet::new(), + requires: vec![ + ComponentDependency { + id: "net.minecraft".into(), + version: None, + }, + ComponentDependency { + id: "net.fabricmc.intermediary".into(), + version: None, + }, + ], + game_jar: None, + main_class: Some(response.mainClass.client), + game_arguments: vec![], + classpath, + downloads, + }; + + fs::write( + out_base.join(format!("{}.json", component.version)), + serde_json::to_string_pretty(&component)?, + )?; + + index.push(component.into()); + } + + index.sort_by(|x, y| y.release_time.cmp(&x.release_time)); + + fs::write( + out_base.join("index.json"), + serde_json::to_string_pretty(&index)?, + )?; + + Ok(()) +} + +async fn get_loader_versions(client: &Client) -> Result> { + let response = client.get("https://maven.quiltmc.org/repository/release/org/quiltmc/quilt-loader/maven-metadata.xml") + .header("User-Agent", "helixlauncher-meta (prototype)") + .send().await? + .text().await?; + let response: Metadata = quick_xml::de::from_str(&response)?; + Ok(response.versioning.versions.version) +} + + +#[derive(Deserialize, Debug)] +struct Libraries { + client: Vec, + common: Vec, + server: Vec, +} + +#[derive(Deserialize, Debug)] +struct MainClass { + client: String, + server: String, + serverLauncher: Option, +} + +#[derive(Deserialize, Debug)] +struct LoaderMeta { + version: i32, + libraries: Libraries, + mainClass: MainClass, +} From 87963e5badc473a877af98ab8ca2c2bce95461aa Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Mon, 24 Jul 2023 00:25:42 +0200 Subject: [PATCH 2/8] Fixes not getting the correct size setting the wrong url (was maven root) Also the first special casing for broken stuff --- src/intermediary.rs | 2 +- src/main.rs | 5 +++-- src/quilt.rs | 6 +++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/intermediary.rs b/src/intermediary.rs index fadbd48..7198a6c 100644 --- a/src/intermediary.rs +++ b/src/intermediary.rs @@ -20,7 +20,7 @@ pub async fn process(client: &Client) -> Result<()> { let library = crate::Library { name: GradleSpecifier::from_str(&format!("net.fabricmc:intermediary:{version}")).unwrap(), url: "https://maven.fabricmc.net/".into() }; let downloads = vec![Download { name: library.name.clone(), - url: library.url.clone(), + url: library.name.to_url(&library.url), hash: get_hash(client, &library).await?, size: get_size(client, &library).await?.try_into().unwrap(), }]; diff --git a/src/main.rs b/src/main.rs index a324a6f..fb367e8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -76,8 +76,9 @@ pub(crate) async fn get_size(client: &Client, coord: &Library) -> Result { .header("User-Agent", "helixlauncher-meta (prototype)") .send() .await? - .content_length() - .expect("Cannot handle servers returning no content length")) + .headers() + .get("content-length") + .expect("Cannot handle servers returning no content length").to_str()?.parse()?) } #[derive(Deserialize, Debug)] diff --git a/src/quilt.rs b/src/quilt.rs index a1354f1..1d27831 100644 --- a/src/quilt.rs +++ b/src/quilt.rs @@ -17,6 +17,10 @@ pub async fn process(client: &Client) -> Result<()> { let mut index: Index = vec![]; for loader_version in get_loader_versions(client).await? { + if loader_version == "0.17.5-beta.4" { // This version's meta is very broken and I hate it + continue; + } + let response = client.get(format!("https://maven.quiltmc.org/repository/release/org/quiltmc/quilt-loader/{loader_version}/quilt-loader-{loader_version}.json")) .header("User-Agent", "helixlauncher-meta (prototype)") .send().await?; @@ -39,7 +43,7 @@ pub async fn process(client: &Client) -> Result<()> { for library in response.libraries.common { downloads.push(Download { name: library.name.clone(), - url: library.url.clone(), + url: library.name.to_url(&library.url), hash: crate::get_hash(client, &library).await?, size: crate::get_size(client, &library).await?.try_into().unwrap(), }); From d9fd594c1001dda9f458fda935c38b7205957f38 Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Mon, 24 Jul 2023 00:39:56 +0200 Subject: [PATCH 3/8] Actually add the loader to downlaods & classpath --- src/quilt.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/quilt.rs b/src/quilt.rs index 1d27831..625656e 100644 --- a/src/quilt.rs +++ b/src/quilt.rs @@ -1,10 +1,10 @@ -use std::{collections::BTreeSet, fs, path::Path}; +use std::{collections::BTreeSet, fs, path::Path, str::FromStr}; use anyhow::{Context, Result}; use chrono::{TimeZone, Utc}; use helixlauncher_meta::{ component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, - index::Index, + index::Index, util::GradleSpecifier, }; use reqwest::Client; use serde::Deserialize; @@ -38,8 +38,14 @@ pub async fn process(client: &Client) -> Result<()> { .context("unable to parse release timestamp")?; let response: LoaderMeta = response.json().await?; - let mut downloads = vec![]; - let mut classpath = vec![]; + let library = crate::Library { name: GradleSpecifier::from_str(&format!("org.quiltmc:quilt-loader:{loader_version}")).unwrap(), url: "https://maven.quiltmc.org/repository/release/".into() }; + let mut downloads = vec![Download { + name: library.name.clone(), + url: library.name.to_url(&library.url), + hash: crate::get_hash(client, &library).await?, + size: crate::get_size(client, &library).await?.try_into().unwrap(), + }]; + let mut classpath = vec![ConditionalClasspathEntry::All(library.name)]; for library in response.libraries.common { downloads.push(Download { name: library.name.clone(), From 404b8c4f18dbe6dc360a73f0f7f4fe26c192bfad Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:52:01 +0200 Subject: [PATCH 4/8] Use meta instead of maven meta (no xml parsing needed anymore) --- Cargo.lock | 11 ----------- Cargo.toml | 1 - src/intermediary.rs | 45 ++++++++++++++++++++++++++++++--------------- src/main.rs | 34 +++++----------------------------- src/quilt.rs | 45 ++++++++++++++++++++++++++++++--------------- 5 files changed, 65 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 676d846..15f6212 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -660,7 +660,6 @@ dependencies = [ "indexmap 2.0.0", "lazy_static", "maven-version-rs", - "quick-xml", "regex", "reqwest", "serde", @@ -1107,16 +1106,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "quick-xml" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b9228215d82c7b61490fec1de287136b5de6f5700f6e58ea9ad61a7964ca51" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quote" version = "1.0.31" diff --git a/Cargo.toml b/Cargo.toml index 99f8ba1..215ae8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ helixlauncher-meta = {path = "helixlauncher-meta"} indexmap = { version = "2.0.0", features = ["serde"] } lazy_static = "1.4.0" maven-version-rs = "0.1.0" -quick-xml = { version = "0.29.0", features = ["serde", "serialize"] } regex = "1.7.3" reqwest = {version = "0.11", features = ["json"]} serde = {version = "1", features = ["derive"]} diff --git a/src/intermediary.rs b/src/intermediary.rs index 7198a6c..26d56a3 100644 --- a/src/intermediary.rs +++ b/src/intermediary.rs @@ -4,11 +4,13 @@ use anyhow::Result; use chrono::Utc; use helixlauncher_meta::{ component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, - index::Index, util::GradleSpecifier, + index::Index, + util::GradleSpecifier, }; use reqwest::Client; +use serde::Deserialize; -use crate::{Metadata, get_hash, get_size}; +use crate::{get_hash, get_size}; pub async fn process(client: &Client) -> Result<()> { let out_base = Path::new("out/net.fabricmc.intermediary"); @@ -17,15 +19,19 @@ pub async fn process(client: &Client) -> Result<()> { let mut index: Index = vec![]; for version in get_versions(client).await? { - let library = crate::Library { name: GradleSpecifier::from_str(&format!("net.fabricmc:intermediary:{version}")).unwrap(), url: "https://maven.fabricmc.net/".into() }; + let library = crate::Library { + name: GradleSpecifier::from_str(&format!("net.fabricmc:intermediary:{version}")) + .unwrap(), + url: "https://maven.fabricmc.net/".into(), + }; let downloads = vec![Download { - name: library.name.clone(), - url: library.name.to_url(&library.url), - hash: get_hash(client, &library).await?, - size: get_size(client, &library).await?.try_into().unwrap(), - }]; + name: library.name.clone(), + url: library.name.to_url(&library.url), + hash: get_hash(client, &library).await?, + size: get_size(client, &library).await?.try_into().unwrap(), + }]; let classpath = vec![ConditionalClasspathEntry::All(library.name)]; - + let component = Component { format_version: 1, assets: None, @@ -66,10 +72,19 @@ pub async fn process(client: &Client) -> Result<()> { } async fn get_versions(client: &Client) -> Result> { - let response = client.get("https://maven.fabricmc.net/net/fabricmc/intermediary/maven-metadata.xml") - .header("User-Agent", "helixlauncher-meta (prototype)") - .send().await? - .text().await?; - let response: Metadata = quick_xml::de::from_str(&response)?; - Ok(response.versioning.versions.version) + let response: Vec = client + .get("https://meta.fabricmc.net/v2/versions/intermediary") + .header("User-Agent", "helixlauncher-meta (prototype)") + .send() + .await? + .json() + .await?; + Ok(response.into_iter().map(|v| v.version).collect()) +} + +#[derive(Deserialize)] +struct IntermediaryVersionData { + maven: GradleSpecifier, + version: String, + stable: bool, } diff --git a/src/main.rs b/src/main.rs index fb367e8..9ef0459 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,10 +5,8 @@ */ #![deny(rust_2018_idioms)] -use std::{fs, path::Path}; - use anyhow::Result; -use helixlauncher_meta::{component::{Component, Hash}, index::Index, util::GradleSpecifier}; +use helixlauncher_meta::{component::Hash, util::GradleSpecifier}; use reqwest::Client; use serde::Deserialize; @@ -34,30 +32,6 @@ async fn main() -> Result<()> { Ok(()) } -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -#[allow(unused)] -pub(crate) struct Metadata { - artifact_id: String, - group_id: String, - versioning: Versioning, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -#[allow(unused)] -pub(crate) struct Versioning { - latest: String, - release: String, - versions: VersionList, - last_updated: String, -} - -#[derive(Deserialize)] -pub(crate) struct VersionList { - version: Vec, -} - pub(crate) async fn get_hash(client: &Client, coord: &Library) -> Result { Ok(Hash::SHA256( client @@ -78,11 +52,13 @@ pub(crate) async fn get_size(client: &Client, coord: &Library) -> Result { .await? .headers() .get("content-length") - .expect("Cannot handle servers returning no content length").to_str()?.parse()?) + .expect("Cannot handle servers returning no content length") + .to_str()? + .parse()?) } #[derive(Deserialize, Debug)] struct Library { name: GradleSpecifier, url: String, -} \ No newline at end of file +} diff --git a/src/quilt.rs b/src/quilt.rs index 625656e..64f8279 100644 --- a/src/quilt.rs +++ b/src/quilt.rs @@ -4,12 +4,13 @@ use anyhow::{Context, Result}; use chrono::{TimeZone, Utc}; use helixlauncher_meta::{ component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, - index::Index, util::GradleSpecifier, + index::Index, + util::GradleSpecifier, }; use reqwest::Client; use serde::Deserialize; -use crate::{Metadata, Library}; +use crate::Library; pub async fn process(client: &Client) -> Result<()> { let out_base = Path::new("out/org.quiltmc.quilt-loader"); fs::create_dir_all(out_base)?; @@ -17,7 +18,8 @@ pub async fn process(client: &Client) -> Result<()> { let mut index: Index = vec![]; for loader_version in get_loader_versions(client).await? { - if loader_version == "0.17.5-beta.4" { // This version's meta is very broken and I hate it + if loader_version == "0.17.5-beta.4" { + // This version's meta is very broken and I hate it continue; } @@ -38,13 +40,17 @@ pub async fn process(client: &Client) -> Result<()> { .context("unable to parse release timestamp")?; let response: LoaderMeta = response.json().await?; - let library = crate::Library { name: GradleSpecifier::from_str(&format!("org.quiltmc:quilt-loader:{loader_version}")).unwrap(), url: "https://maven.quiltmc.org/repository/release/".into() }; + let library = crate::Library { + name: GradleSpecifier::from_str(&format!("org.quiltmc:quilt-loader:{loader_version}")) + .unwrap(), + url: "https://maven.quiltmc.org/repository/release/".into(), + }; let mut downloads = vec![Download { - name: library.name.clone(), - url: library.name.to_url(&library.url), - hash: crate::get_hash(client, &library).await?, - size: crate::get_size(client, &library).await?.try_into().unwrap(), - }]; + name: library.name.clone(), + url: library.name.to_url(&library.url), + hash: crate::get_hash(client, &library).await?, + size: crate::get_size(client, &library).await?.try_into().unwrap(), + }]; let mut classpath = vec![ConditionalClasspathEntry::All(library.name)]; for library in response.libraries.common { downloads.push(Download { @@ -102,14 +108,23 @@ pub async fn process(client: &Client) -> Result<()> { } async fn get_loader_versions(client: &Client) -> Result> { - let response = client.get("https://maven.quiltmc.org/repository/release/org/quiltmc/quilt-loader/maven-metadata.xml") - .header("User-Agent", "helixlauncher-meta (prototype)") - .send().await? - .text().await?; - let response: Metadata = quick_xml::de::from_str(&response)?; - Ok(response.versioning.versions.version) + let response: Vec = client + .get("https://meta.quiltmc.org/v3/versions/loader") + .header("User-Agent", "helixlauncher-meta (prototype)") + .send() + .await? + .json() + .await?; + Ok(response.into_iter().map(|x| x.version).collect()) } +#[derive(Deserialize, Debug)] +struct LoaderVersionData { + separator: String, + build: i32, + maven: GradleSpecifier, + version: String, +} #[derive(Deserialize, Debug)] struct Libraries { From db535e20728bfac3ad85803bd5d417f709d590c1 Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Wed, 26 Jul 2023 18:16:58 +0200 Subject: [PATCH 5/8] Fix: Intermediary used Utc::now as release date --- src/intermediary.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/intermediary.rs b/src/intermediary.rs index 26d56a3..2074032 100644 --- a/src/intermediary.rs +++ b/src/intermediary.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeSet, fs, path::Path, str::FromStr}; use anyhow::Result; -use chrono::Utc; +use chrono::DateTime; use helixlauncher_meta::{ component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, index::Index, @@ -30,6 +30,25 @@ pub async fn process(client: &Client) -> Result<()> { hash: get_hash(client, &library).await?, size: get_size(client, &library).await?.try_into().unwrap(), }]; + + let release_time = DateTime::parse_from_rfc2822( + // TODO: This does one more request than necessary, should get_size or get_hash be merged into this? + client + .head(library.name.to_url(&library.url)) + .header("User-Agent", "helixlauncher-meta (prototype)") + .send() + .await? + .headers() + .get("last-modified") + .expect("Cannot handle servers returning no last-modified") + .to_str()?, + ) + .expect(&format!( + "Error parsing last-modified header of {}", + library.name.to_url(&library.url) + )) + .into(); + let classpath = vec![ConditionalClasspathEntry::All(library.name)]; let component = Component { @@ -39,7 +58,7 @@ pub async fn process(client: &Client) -> Result<()> { id: "net.fabricmc.intermediary".into(), jarmods: vec![], natives: vec![], - release_time: Utc::now(), + release_time, version: version.clone(), traits: BTreeSet::new(), requires: vec![ComponentDependency { From aba2899ea8737db48d650c49879d46e0e8ea7ec6 Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Thu, 27 Jul 2023 21:34:45 +0200 Subject: [PATCH 6/8] Update User-Agent header --- src/intermediary.rs | 4 ++-- src/main.rs | 4 ++-- src/quilt.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/intermediary.rs b/src/intermediary.rs index 2074032..7dfa304 100644 --- a/src/intermediary.rs +++ b/src/intermediary.rs @@ -35,7 +35,7 @@ pub async fn process(client: &Client) -> Result<()> { // TODO: This does one more request than necessary, should get_size or get_hash be merged into this? client .head(library.name.to_url(&library.url)) - .header("User-Agent", "helixlauncher-meta (prototype)") + .header("User-Agent", "helixlauncher-meta") .send() .await? .headers() @@ -93,7 +93,7 @@ pub async fn process(client: &Client) -> Result<()> { async fn get_versions(client: &Client) -> Result> { let response: Vec = client .get("https://meta.fabricmc.net/v2/versions/intermediary") - .header("User-Agent", "helixlauncher-meta (prototype)") + .header("User-Agent", "helixlauncher-meta") .send() .await? .json() diff --git a/src/main.rs b/src/main.rs index 9ef0459..99750c3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,7 +36,7 @@ pub(crate) async fn get_hash(client: &Client, coord: &Library) -> Result { Ok(Hash::SHA256( client .get(coord.name.to_url(&coord.url) + ".sha256") - .header("User-Agent", "helixlauncher-meta (prototype)") + .header("User-Agent", "helixlauncher-meta") .send() .await? .text() @@ -47,7 +47,7 @@ pub(crate) async fn get_hash(client: &Client, coord: &Library) -> Result { pub(crate) async fn get_size(client: &Client, coord: &Library) -> Result { Ok(client .head(coord.name.to_url(&coord.url)) - .header("User-Agent", "helixlauncher-meta (prototype)") + .header("User-Agent", "helixlauncher-meta") .send() .await? .headers() diff --git a/src/quilt.rs b/src/quilt.rs index 64f8279..1d79c6a 100644 --- a/src/quilt.rs +++ b/src/quilt.rs @@ -24,7 +24,7 @@ pub async fn process(client: &Client) -> Result<()> { } let response = client.get(format!("https://maven.quiltmc.org/repository/release/org/quiltmc/quilt-loader/{loader_version}/quilt-loader-{loader_version}.json")) - .header("User-Agent", "helixlauncher-meta (prototype)") + .header("User-Agent", "helixlauncher-meta") .send().await?; let release_time = Utc @@ -110,7 +110,7 @@ pub async fn process(client: &Client) -> Result<()> { async fn get_loader_versions(client: &Client) -> Result> { let response: Vec = client .get("https://meta.quiltmc.org/v3/versions/loader") - .header("User-Agent", "helixlauncher-meta (prototype)") + .header("User-Agent", "helixlauncher-meta") .send() .await? .json() From 1776f3765faa55ceabf7d88ba1b9b402c3a20581 Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Sat, 19 Aug 2023 22:36:32 +0200 Subject: [PATCH 7/8] Rework the quilt meta generation: - Lots of async (It felt like caching isnt even needed anymore) - Lots of caching (Best case only the request for update checking is needed) --- src/main.rs | 12 ++- src/quilt.rs | 202 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 156 insertions(+), 58 deletions(-) diff --git a/src/main.rs b/src/main.rs index 99750c3..cc81d4d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,9 +6,10 @@ #![deny(rust_2018_idioms)] use anyhow::Result; +use futures::try_join; use helixlauncher_meta::{component::Hash, util::GradleSpecifier}; use reqwest::Client; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; mod forge; mod intermediary; @@ -19,13 +20,16 @@ mod quilt; async fn main() -> Result<()> { let client = reqwest::Client::new(); - mojang::fetch(&client).await?; + try_join!( + mojang::fetch(&client), + quilt::fetch(&client) + )?; mojang::process()?; // forge::process()?; - quilt::process(&client).await?; + quilt::process()?; intermediary::process(&client).await?; @@ -57,7 +61,7 @@ pub(crate) async fn get_size(client: &Client, coord: &Library) -> Result { .parse()?) } -#[derive(Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] struct Library { name: GradleSpecifier, url: String, diff --git a/src/quilt.rs b/src/quilt.rs index 1d79c6a..f0d6017 100644 --- a/src/quilt.rs +++ b/src/quilt.rs @@ -1,66 +1,152 @@ -use std::{collections::BTreeSet, fs, path::Path, str::FromStr}; +use std::{ + collections::BTreeSet, + fs, iter, + path::{Path, PathBuf}, + str::FromStr, +}; use anyhow::{Context, Result}; -use chrono::{TimeZone, Utc}; +use chrono::{DateTime, TimeZone, Utc}; +use futures::{stream, StreamExt, TryStreamExt}; use helixlauncher_meta::{ component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, index::Index, util::GradleSpecifier, }; use reqwest::Client; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::Library; -pub async fn process(client: &Client) -> Result<()> { - let out_base = Path::new("out/org.quiltmc.quilt-loader"); - fs::create_dir_all(out_base)?; - let mut index: Index = vec![]; +const CONCURRENT_FETCH_LIMIT: usize = 5; + +pub async fn fetch(client: &Client) -> Result<()> { + let upstream_base = Path::new("upstream/quilt"); + let versions_base = upstream_base.join("versions"); + let downloads_base = upstream_base.join("downloads"); + + fs::create_dir_all(&versions_base).unwrap(); + fs::create_dir_all(&downloads_base).unwrap(); + + stream::iter(get_loader_versions(client).await?) + .map(|loader_version| async { + let version_meta = fetch_version(&loader_version, client, &versions_base).await?; + if let Some(version_meta) = version_meta { + fetch_downloads(loader_version, version_meta, client, &downloads_base).await + } else { + Ok(()) + } + }) + .buffer_unordered(CONCURRENT_FETCH_LIMIT) + .try_collect::<()>() + .await?; + Ok(()) +} + +async fn fetch_version( + loader_version: &String, + client: &Client, + versions_base: &PathBuf, +) -> Result> { + if loader_version == "0.17.5-beta.4" { + // This version's meta is very broken and I hate it + return Ok(None); + } - for loader_version in get_loader_versions(client).await? { - if loader_version == "0.17.5-beta.4" { - // This version's meta is very broken and I hate it - continue; - } + let version_path = versions_base.join(format!("{}.json", loader_version)); + if version_path.try_exists()? { + return Ok(Some(serde_json::from_str(&fs::read_to_string( + version_path, + )?)?)); + } - let response = client.get(format!("https://maven.quiltmc.org/repository/release/org/quiltmc/quilt-loader/{loader_version}/quilt-loader-{loader_version}.json")) + let response = client.get(format!("https://maven.quiltmc.org/repository/release/org/quiltmc/quilt-loader/{loader_version}/quilt-loader-{loader_version}.json")) .header("User-Agent", "helixlauncher-meta") .send().await?; - let release_time = Utc - .timestamp_millis_opt( - response - .headers() - .get("quilt-last-modified-timestamp") - .context("Error quilt did not provide release date in metadata")? - .to_str()? - .parse()?, - ) - .single() - .context("unable to parse release timestamp")?; - - let response: LoaderMeta = response.json().await?; - let library = crate::Library { + let release_time = Utc + .timestamp_millis_opt( + response + .headers() + .get("quilt-last-modified-timestamp") + .context("Error quilt did not provide release date in metadata")? + .to_str()? + .parse()?, + ) + .single() + .context("unable to parse release timestamp")?; + + let response: LoaderMeta = response.json().await?; + let response = LoaderMetaWithReleaseTime { + meta: response, + release_time, + }; + + serde_json::to_writer_pretty(fs::File::create(version_path)?, &response)?; + Ok(Some(response)) +} + +async fn fetch_downloads( + loader_version: String, + loader_meta: LoaderMetaWithReleaseTime, + client: &Client, + downloads_base: &PathBuf, +) -> Result<()> { + let downloads_path = downloads_base.join(format!("{}.json", loader_version)); + if downloads_path.try_exists()? { + return Ok(()); + } + + let libraries = loader_meta + .meta + .libraries + .common + .into_iter() + .chain(iter::once(crate::Library { name: GradleSpecifier::from_str(&format!("org.quiltmc:quilt-loader:{loader_version}")) .unwrap(), url: "https://maven.quiltmc.org/repository/release/".into(), - }; - let mut downloads = vec![Download { - name: library.name.clone(), - url: library.name.to_url(&library.url), - hash: crate::get_hash(client, &library).await?, - size: crate::get_size(client, &library).await?.try_into().unwrap(), - }]; - let mut classpath = vec![ConditionalClasspathEntry::All(library.name)]; - for library in response.libraries.common { - downloads.push(Download { - name: library.name.clone(), - url: library.name.to_url(&library.url), - hash: crate::get_hash(client, &library).await?, - size: crate::get_size(client, &library).await?.try_into().unwrap(), - }); - classpath.push(ConditionalClasspathEntry::All(library.name)) - } + })); + + let downloads = stream::iter(libraries) + .map(|library| library_to_download(client, library)) + .buffer_unordered(CONCURRENT_FETCH_LIMIT) + .try_collect::>() + .await?; + + serde_json::to_writer_pretty(fs::File::create(downloads_path)?, &downloads)?; + + Ok(()) +} + +async fn library_to_download(client: &Client, library: Library) -> Result { + Ok(Download { + name: library.name.clone(), + url: library.name.to_url(&library.url), + hash: crate::get_hash(client, &library).await?, + size: crate::get_size(client, &library).await?.try_into().unwrap(), + }) +} + +pub fn process() -> Result<()> { + let upstream_base = Path::new("upstream/quilt"); + let versions_base = upstream_base.join("versions"); + let downloads_base = upstream_base.join("downloads"); + let out_base = Path::new("out/org.quiltmc.quilt-loader"); + fs::create_dir_all(out_base)?; + + let mut index: Index = vec![]; + + for loader_meta in fs::read_dir(versions_base)? { + let loader_meta = loader_meta?; + let loader_version = loader_meta.file_name().clone().to_string_lossy() + [..loader_meta.file_name().len() - 5] + .to_string(); + let downloads: Vec = serde_json::from_str(&fs::read_to_string(&downloads_base.join(loader_meta.file_name()))?)?; + let loader_meta: LoaderMetaWithReleaseTime = + serde_json::from_str(&fs::read_to_string(loader_meta.path())?)?; + + let classpath = downloads.iter().map(|download| ConditionalClasspathEntry::All(download.name.clone())).collect(); let component = Component { format_version: 1, @@ -69,7 +155,7 @@ pub async fn process(client: &Client) -> Result<()> { id: "org.quiltmc.quilt-loader".into(), jarmods: vec![], natives: vec![], - release_time, + release_time: loader_meta.release_time, version: loader_version, traits: BTreeSet::new(), requires: vec![ @@ -83,7 +169,7 @@ pub async fn process(client: &Client) -> Result<()> { }, ], game_jar: None, - main_class: Some(response.mainClass.client), + main_class: Some(loader_meta.meta.main_class.client), game_arguments: vec![], classpath, downloads, @@ -120,29 +206,37 @@ async fn get_loader_versions(client: &Client) -> Result> { #[derive(Deserialize, Debug)] struct LoaderVersionData { - separator: String, - build: i32, - maven: GradleSpecifier, + // separator: String, + // build: i32, + // maven: GradleSpecifier, version: String, } -#[derive(Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] struct Libraries { client: Vec, common: Vec, server: Vec, } -#[derive(Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] struct MainClass { client: String, server: String, - serverLauncher: Option, + #[serde(rename="serverLauncher")] + server_launcher: Option, } -#[derive(Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] struct LoaderMeta { version: i32, libraries: Libraries, - mainClass: MainClass, + #[serde(rename="mainClass")] + main_class: MainClass, +} + +#[derive(Serialize, Deserialize, Debug)] +struct LoaderMetaWithReleaseTime { + meta: LoaderMeta, + release_time: DateTime, } From 5e186d72193b271978da4c57c02bb399be5357d9 Mon Sep 17 00:00:00 2001 From: anonymous123-code <61744596+anonymous123-code@users.noreply.github.com> Date: Sat, 19 Aug 2023 23:02:05 +0200 Subject: [PATCH 8/8] Rework the intermediary meta generation: - Lots of async - Lots of caching (Best case only the request for update checking is needed) - cargo fmt --- src/intermediary.rs | 124 +++++++++++++++++++++++++++++--------------- src/main.rs | 5 +- src/quilt.rs | 13 +++-- 3 files changed, 95 insertions(+), 47 deletions(-) diff --git a/src/intermediary.rs b/src/intermediary.rs index 7dfa304..19ca3e8 100644 --- a/src/intermediary.rs +++ b/src/intermediary.rs @@ -1,55 +1,91 @@ use std::{collections::BTreeSet, fs, path::Path, str::FromStr}; use anyhow::Result; -use chrono::DateTime; +use chrono::{DateTime, Utc}; +use futures::{stream, StreamExt, TryStreamExt}; use helixlauncher_meta::{ component::{Component, ComponentDependency, ConditionalClasspathEntry, Download}, index::Index, util::GradleSpecifier, }; use reqwest::Client; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::{get_hash, get_size}; -pub async fn process(client: &Client) -> Result<()> { +const CONCURRENT_FETCH_LIMIT: usize = 5; +pub async fn fetch(client: &Client) -> Result<()> { + let upstream_base = Path::new("upstream/intermediary"); + + fs::create_dir_all(&upstream_base).unwrap(); + + stream::iter(get_versions(client).await?) + .map(|version| async { fetch_version(version, client, &upstream_base).await }) + .buffer_unordered(CONCURRENT_FETCH_LIMIT) + .try_collect::<()>() + .await?; + Ok(()) +} + +async fn fetch_version(version: String, client: &Client, upstream_base: &Path) -> Result<()> { + let version_path = upstream_base.join(format!("{}.json", version)); + if version_path.try_exists()? { + return Ok(()); + } + + let library = crate::Library { + name: GradleSpecifier::from_str(&format!("net.fabricmc:intermediary:{version}")).unwrap(), + url: "https://maven.fabricmc.net/".into(), + }; + let download = Download { + name: library.name.clone(), + url: library.name.to_url(&library.url), + hash: get_hash(client, &library).await?, + size: get_size(client, &library).await?.try_into().unwrap(), + }; + + let release_time = DateTime::parse_from_rfc2822( + // TODO: This does one more request than necessary, should get_size or get_hash be merged into this? + client + .head(library.name.to_url(&library.url)) + .header("User-Agent", "helixlauncher-meta") + .send() + .await? + .headers() + .get("last-modified") + .expect("Cannot handle servers returning no last-modified") + .to_str()?, + ) + .expect(&format!( + "Error parsing last-modified header of {}", + library.name.to_url(&library.url) + )) + .into(); + + let download = DownloadWithReleaseTime { + download, + release_time, + }; + + fs::write(version_path, serde_json::to_string_pretty(&download)?)?; + + Ok(()) +} + +pub fn process() -> Result<()> { let out_base = Path::new("out/net.fabricmc.intermediary"); + let upstream_base = Path::new("upstream/intermediary"); fs::create_dir_all(out_base)?; let mut index: Index = vec![]; - for version in get_versions(client).await? { - let library = crate::Library { - name: GradleSpecifier::from_str(&format!("net.fabricmc:intermediary:{version}")) - .unwrap(), - url: "https://maven.fabricmc.net/".into(), - }; - let downloads = vec![Download { - name: library.name.clone(), - url: library.name.to_url(&library.url), - hash: get_hash(client, &library).await?, - size: get_size(client, &library).await?.try_into().unwrap(), - }]; - - let release_time = DateTime::parse_from_rfc2822( - // TODO: This does one more request than necessary, should get_size or get_hash be merged into this? - client - .head(library.name.to_url(&library.url)) - .header("User-Agent", "helixlauncher-meta") - .send() - .await? - .headers() - .get("last-modified") - .expect("Cannot handle servers returning no last-modified") - .to_str()?, - ) - .expect(&format!( - "Error parsing last-modified header of {}", - library.name.to_url(&library.url) - )) - .into(); - - let classpath = vec![ConditionalClasspathEntry::All(library.name)]; + for version_meta in fs::read_dir(upstream_base)? { + let version_meta: DownloadWithReleaseTime = + serde_json::from_str(&fs::read_to_string(version_meta?.path())?)?; + + let classpath = vec![ConditionalClasspathEntry::All( + version_meta.download.name.clone(), + )]; let component = Component { format_version: 1, @@ -58,18 +94,18 @@ pub async fn process(client: &Client) -> Result<()> { id: "net.fabricmc.intermediary".into(), jarmods: vec![], natives: vec![], - release_time, - version: version.clone(), + release_time: version_meta.release_time, + version: version_meta.download.name.version.clone(), traits: BTreeSet::new(), requires: vec![ComponentDependency { id: "net.minecraft".into(), - version: Some(version), + version: Some(version_meta.download.name.version.clone()), }], game_jar: None, main_class: None, game_arguments: vec![], classpath, - downloads, + downloads: vec![version_meta.download], }; fs::write( @@ -103,7 +139,13 @@ async fn get_versions(client: &Client) -> Result> { #[derive(Deserialize)] struct IntermediaryVersionData { - maven: GradleSpecifier, + // maven: GradleSpecifier, version: String, - stable: bool, + // stable: bool, +} + +#[derive(Serialize, Deserialize)] +struct DownloadWithReleaseTime { + download: Download, + release_time: DateTime, } diff --git a/src/main.rs b/src/main.rs index cc81d4d..9c5d34d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,8 @@ async fn main() -> Result<()> { try_join!( mojang::fetch(&client), - quilt::fetch(&client) + quilt::fetch(&client), + intermediary::fetch(&client), )?; mojang::process()?; @@ -31,7 +32,7 @@ async fn main() -> Result<()> { quilt::process()?; - intermediary::process(&client).await?; + intermediary::process()?; Ok(()) } diff --git a/src/quilt.rs b/src/quilt.rs index f0d6017..86aa7fd 100644 --- a/src/quilt.rs +++ b/src/quilt.rs @@ -142,11 +142,16 @@ pub fn process() -> Result<()> { let loader_version = loader_meta.file_name().clone().to_string_lossy() [..loader_meta.file_name().len() - 5] .to_string(); - let downloads: Vec = serde_json::from_str(&fs::read_to_string(&downloads_base.join(loader_meta.file_name()))?)?; + let downloads: Vec = serde_json::from_str(&fs::read_to_string( + &downloads_base.join(loader_meta.file_name()), + )?)?; let loader_meta: LoaderMetaWithReleaseTime = serde_json::from_str(&fs::read_to_string(loader_meta.path())?)?; - let classpath = downloads.iter().map(|download| ConditionalClasspathEntry::All(download.name.clone())).collect(); + let classpath = downloads + .iter() + .map(|download| ConditionalClasspathEntry::All(download.name.clone())) + .collect(); let component = Component { format_version: 1, @@ -223,7 +228,7 @@ struct Libraries { struct MainClass { client: String, server: String, - #[serde(rename="serverLauncher")] + #[serde(rename = "serverLauncher")] server_launcher: Option, } @@ -231,7 +236,7 @@ struct MainClass { struct LoaderMeta { version: i32, libraries: Libraries, - #[serde(rename="mainClass")] + #[serde(rename = "mainClass")] main_class: MainClass, }