From 2b79f0d5464adb332b9bc1f42db7daabb49fc96a Mon Sep 17 00:00:00 2001 From: atandrewlee Date: Mon, 2 Jun 2025 13:01:09 -0400 Subject: [PATCH 1/2] removed commented code --- cli/src/commands/init.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cli/src/commands/init.rs b/cli/src/commands/init.rs index 686d15e..b1a8d70 100644 --- a/cli/src/commands/init.rs +++ b/cli/src/commands/init.rs @@ -73,14 +73,6 @@ pub fn init_command(args: &InitArgs) -> Result<()> { } }; - /* - let dependency_tree: DependencyTreeNode = find_dependency_tree( - DependencyTreeRequest::GitProject { - url: args.from_git.clone(), - branch: args.checkout.clone(), - } - ).map_err(|e| Error::msg(e))?; - */ let nb_dependencies: usize = dependency_tree.flatten().dependencies.len(); println!(" > {} unique dependencies were found in the project", nb_dependencies); println!(" > Saving dependency tree"); From 170a08baab47e638529431c55f8d05afd6a4b471 Mon Sep 17 00:00:00 2001 From: atandrewlee Date: Mon, 2 Jun 2025 13:40:41 -0400 Subject: [PATCH 2/2] added timer for failed requests --- .../go_dependency_tree_generator.rs | 51 ++++++++++++++----- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/dependency-analysis/src/dependency_tree_generators/go_dependency_tree_generator.rs b/dependency-analysis/src/dependency_tree_generators/go_dependency_tree_generator.rs index b5aa0e4..ac6d2ca 100644 --- a/dependency-analysis/src/dependency_tree_generators/go_dependency_tree_generator.rs +++ b/dependency-analysis/src/dependency_tree_generators/go_dependency_tree_generator.rs @@ -1,7 +1,10 @@ use std::collections::{HashMap, HashSet}; +use std::time::Duration; +use std::thread::sleep; use anyhow::{Error, Result}; -use reqwest::blocking::get; +use reqwest::blocking::{Client}; +use reqwest::StatusCode; use scraper::{Html, Selector}; use source_wand_common::{project::Project, project_manipulator::project_manipulator::ProjectManipulator}; @@ -121,19 +124,43 @@ fn extract_repository_url(module_path: &str, cache: &mut HashMap fn resolve_vanity_import(module_path: &str) -> Option { let url: String = format!("https://{}?go-get=1", module_path); - - let document: Html = Html::parse_document(&get(&url).ok()?.text().ok()?); - let selector: Selector = Selector::parse(r#"meta[name="go-import"]"#).ok()?; - - for element in document.select(&selector) { - if let Some(content) = element.value().attr("content") { - let parts: Vec<&str> = content.split_whitespace().collect(); - - if parts.len() == 3 { - return Some(parts[2].to_string()); + let client = Client::new(); + let max_retries = 3; + let base_delay = Duration::from_secs(1); + + for attempt in 0..max_retries { + match client.get(&url).send() { + Ok(response) => { + if response.status() == StatusCode::TOO_MANY_REQUESTS { + let delay = base_delay * 2u32.pow(attempt); + sleep(delay); + continue; + } + let text = response.text().ok()?; + let document: Html = Html::parse_document(&text); + let selector: Selector = Selector::parse(r#"meta[name="go-import"]"#).ok()?; + + for element in document.select(&selector) { + if let Some(content) = element.value().attr("content") { + let parts: Vec<&str> = content.split_whitespace().collect(); + + if parts.len() == 3 { + return Some(parts[2].to_string()); + } + } + } + } + Err(e) => { + if attempt == max_retries - 1 { + println!("Failed after {} retries: {}", max_retries, e); + return None; + } + // Calculate delay with exponential backoff + let delay = base_delay * 2u32.pow(attempt); + println!("Request error, retrying in {:?}: {}", delay, e); + sleep(delay); } } } - None }