diff --git a/Cargo.lock b/Cargo.lock index 998ea65c6..e3626d3e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a026259da4f1a13b4af60cda453c392de64c58c12d239c560923e0382f42f2b9" dependencies = [ - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "rustc_version", "smol_str", @@ -2757,7 +2757,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.12", ] [[package]] @@ -2771,7 +2771,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.12", ] [[package]] @@ -3751,7 +3751,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" dependencies = [ "atomic", - "parking_lot", + "parking_lot 0.12.5", "pear", "serde", "tempfile", @@ -3999,7 +3999,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot", + "parking_lot 0.12.5", ] [[package]] @@ -4674,7 +4674,7 @@ dependencies = [ "ipconfig", "moka", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "rand 0.9.2", "resolv-conf", "smallvec", @@ -6273,7 +6273,7 @@ checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ "bitflags 2.10.0", "libc", - "redox_syscall", + "redox_syscall 0.5.18", ] [[package]] @@ -6721,7 +6721,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "equivalent", - "parking_lot", + "parking_lot 0.12.5", "portable-atomic", "rustc_version", "smallvec", @@ -7735,6 +7735,7 @@ dependencies = [ "orb-update-agent-dbus", "reqwest 0.12.24", "reqwest-middleware", + "reqwest-retry", "reqwest-tracing", "serde", "serde_json", @@ -8764,6 +8765,17 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -8771,7 +8783,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -8782,7 +8808,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link 0.2.1", ] @@ -9527,7 +9553,7 @@ dependencies = [ "jep106", "nusb 0.1.14", "object 0.36.7", - "parking_lot", + "parking_lot 0.12.5", "probe-rs-target", "rmp-serde", "scroll", @@ -10117,6 +10143,15 @@ dependencies = [ "libc", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -10324,6 +10359,28 @@ dependencies = [ "tower-service", ] +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom 0.2.16", + "http 1.3.1", + "hyper 1.7.0", + "parking_lot 0.11.2", + "reqwest 0.12.24", + "reqwest-middleware", + "retry-policies", + "thiserror 1.0.69", + "tokio", + "tracing", + "wasm-timer", +] + [[package]] name = "reqwest-tracing" version = "0.5.8" @@ -10357,6 +10414,15 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "rfc6979" version = "0.3.1" @@ -11280,7 +11346,7 @@ dependencies = [ "futures", "lazy_static", "log", - "parking_lot", + "parking_lot 0.12.5", "serial_test_derive 2.0.0", ] @@ -11293,7 +11359,7 @@ dependencies = [ "futures", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.5", "scc", "serial_test_derive 3.2.0", ] @@ -11892,7 +11958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" dependencies = [ "new_debug_unreachable", - "parking_lot", + "parking_lot 0.12.5", "phf_shared 0.11.3", "precomputed-hash", ] @@ -12023,7 +12089,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ea7b4bfbd3d9980392cd9f90e4158212a5f775fa58e9b85216a0bf739067d" dependencies = [ "hex", - "parking_lot", + "parking_lot 0.12.5", "pnet_packet", "rand 0.9.2", "socket2 0.6.1", @@ -12536,7 +12602,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", + "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", "socket2 0.6.1", @@ -13603,6 +13669,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.82" diff --git a/orb-backend-status/Cargo.toml b/orb-backend-status/Cargo.toml index f3dfb914c..110e5cf39 100644 --- a/orb-backend-status/Cargo.toml +++ b/orb-backend-status/Cargo.toml @@ -31,6 +31,7 @@ orb-update-agent-dbus.workspace = true reqwest = { workspace = true, features = ["json"] } reqwest-middleware = { version = "0.4.1", features = ["json"] } reqwest-tracing = { version = "0.5.6", features = ["opentelemetry_0_27"] } +reqwest-retry = "0.7.0" serde = { workspace = true, features = ["derive"] } serde_json.workspace = true serde_with = { version = "3.11.0" } diff --git a/orb-backend-status/src/backend/status.rs b/orb-backend-status/src/backend/status.rs index 2cc2c4c2c..388ce6ce0 100644 --- a/orb-backend-status/src/backend/status.rs +++ b/orb-backend-status/src/backend/status.rs @@ -4,6 +4,7 @@ use orb_endpoints::{v2::Endpoints as EndpointsV2, Backend}; use orb_info::{OrbId, OrbJabilId, OrbName}; use reqwest::Url; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, Extension}; +use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; use reqwest_tracing::{OtelName, TracingMiddleware}; use std::{str::FromStr, time::Duration}; use tokio::sync::watch; @@ -54,15 +55,24 @@ impl StatusClient { ) -> Result { let orb_os_version = orb_os_version()?; info!("backend-status orb_os_version: {}", orb_os_version); + + let retry_policy = ExponentialBackoff::builder() + .retry_bounds(Duration::from_millis(100), Duration::from_secs(2)) + .build_with_max_retries(5); + let reqwest_client = reqwest::Client::builder() - .timeout(Duration::from_secs(60)) + .connect_timeout(Duration::from_secs(2)) + .timeout(Duration::from_secs(3)) .user_agent("orb-backend-status") .build() .expect("Failed to build client"); + let name = orb_id.as_str().to_string().into(); + let client = ClientBuilder::new(reqwest_client) .with_init(Extension(OtelName(name))) .with(TracingMiddleware::default()) + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) .build(); let backend = match Backend::from_str(&args.backend) { diff --git a/orb-connd/src/telemetry/backend_status_wifi_reporter.rs b/orb-connd/src/telemetry/backend_status_wifi_reporter.rs index 34b002d11..937ab02e9 100644 --- a/orb-connd/src/telemetry/backend_status_wifi_reporter.rs +++ b/orb-connd/src/telemetry/backend_status_wifi_reporter.rs @@ -59,25 +59,33 @@ async fn run_reporter( interval.set_missed_tick_behavior(time::MissedTickBehavior::Skip); loop { - tokio::select! { + let scan = tokio::select! { _ = state_stream.next() => { info!("NetworkManager state changed - sending immediate WiFi status"); + false } _ = primary_conn_stream.next() => { info!("Primary connection changed - sending immediate WiFi status"); + false } - _ = interval.tick() => {} + _ = interval.tick() => { + true + } }; - if let Err(e) = report(&nm, &session_bus).await { + if let Err(e) = report(&nm, &session_bus, scan).await { error!("failed to report to backend status: {e}"); } } } -async fn report(nm: &NetworkManager, session_bus: &zbus::Connection) -> Result<()> { +async fn report( + nm: &NetworkManager, + session_bus: &zbus::Connection, + scan: bool, +) -> Result<()> { let be_status = BackendStatusProxy::new(session_bus) .await .wrap_err("Failed to create Backend Status dbus Proxy")?; @@ -106,19 +114,22 @@ async fn report(nm: &NetworkManager, session_bus: &zbus::Connection) -> Result<( }) .collect(); - let scanned_networks: Vec = nm - .wifi_scan() - .await - .inspect_err(|e| warn!("failed to scan wifi: {e}")) - .unwrap_or_default() - .into_iter() - .map(|ap| WifiNetwork { - bssid: ap.bssid, - ssid: ap.ssid, - frequency: ap.freq_mhz, - signal_level: ap.rssi.unwrap_or_default(), - }) - .collect(); + let scanned_networks = match scan { + true => nm + .wifi_scan() + .await + .inspect_err(|e| warn!("failed to scan wifi: {e}")) + .unwrap_or_default() + .into_iter() + .map(|ap| WifiNetwork { + bssid: ap.bssid, + ssid: ap.ssid, + frequency: ap.freq_mhz, + signal_level: ap.rssi.unwrap_or_default(), + }) + .collect(), + false => Vec::new(), + }; let _ = async { be_status diff --git a/orb-jobs-agent/src/job_system/handler.rs b/orb-jobs-agent/src/job_system/handler.rs index 37cd03427..ddc3838bf 100644 --- a/orb-jobs-agent/src/job_system/handler.rs +++ b/orb-jobs-agent/src/job_system/handler.rs @@ -133,9 +133,10 @@ impl JobHandler { .namespace(relay_namespace) .auth(auth.clone()) .connection_timeout(Duration::from_secs(5)) - .connection_backoff(Duration::from_secs(3)) - .keep_alive_interval(Duration::from_secs(10)) - .keep_alive_timeout(Duration::from_secs(5)) + .connection_backoff(Duration::from_millis(500)) + .keep_alive_interval(Duration::from_secs(2)) + .keep_alive_timeout(Duration::from_secs(2)) + .ack_timeout(Duration::from_secs(5)) .build(); info!("Connecting to relay: {:?}", relay_host);