From 5601d8706238151b6580458f643ed184fbf6d6b6 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Wed, 3 Dec 2025 20:29:51 +0000 Subject: [PATCH 1/9] WIP: allow launching multiple racks with omicron-dev --- dev-tools/omicron-dev/src/main.rs | 256 ++++++++++++++++++------------ 1 file changed, 152 insertions(+), 104 deletions(-) diff --git a/dev-tools/omicron-dev/src/main.rs b/dev-tools/omicron-dev/src/main.rs index 7ba618758b9..8e027c27afa 100644 --- a/dev-tools/omicron-dev/src/main.rs +++ b/dev-tools/omicron-dev/src/main.rs @@ -57,6 +57,9 @@ struct RunAllArgs { /// Override the nexus configuration file. #[clap(long, default_value = DEFAULT_NEXUS_CONFIG)] nexus_config: Utf8PathBuf, + /// Number of "racks" to launch + #[clap(long, default_value_t = 1)] + count: u8, } impl RunAllArgs { @@ -78,121 +81,166 @@ impl RunAllArgs { if_exists: dropshot::ConfigLoggingIfExists::Fail, }; - if let Some(p) = self.nexus_listen_port { - config - .deployment - .dropshot_external - .dropshot - .bind_address - .set_port(p); - } - - println!("omicron-dev: setting up all services ... "); - let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< - omicron_nexus::Server, - >(&mut config, 0, self.gateway_config.clone()) - .await - .context("error setting up services")?; - - println!("omicron-dev: Adding disks to first sled agent"); - - // This is how our integration tests are identifying that "disks exist" - // within the database. - // - // This inserts: - // - DEFAULT_ZPOOL_COUNT zpools, each of which contains: - // - A crucible dataset - // - A debug dataset - DiskTest::new(&cptestctx).await; - - println!("omicron-dev: services are running."); - - // Print out basic information about what was started. - // NOTE: The stdout strings here are not intended to be stable, but they - // are used by the test suite. - let addr = cptestctx.external_client.bind_address; - println!("omicron-dev: nexus external API: {:?}", addr); - println!( - "omicron-dev: nexus internal API: {:?}", - cptestctx.server.get_http_server_internal_address(), - ); - println!( - "omicron-dev: nexus lockstep API: {:?}", - cptestctx.server.get_http_server_lockstep_address(), - ); - println!( - "omicron-dev: cockroachdb pid: {}", - cptestctx.database.pid(), - ); - println!( - "omicron-dev: cockroachdb URL: {}", - cptestctx.database.pg_config() - ); - println!( - "omicron-dev: cockroachdb directory: {}", - cptestctx.database.temp_dir().display() - ); - println!( - "omicron-dev: clickhouse native addr: {}", - cptestctx.clickhouse.native_address(), - ); - println!( - "omicron-dev: clickhouse http addr: {}", - cptestctx.clickhouse.http_address(), - ); - println!( - "omicron-dev: internal DNS HTTP: http://{}", - cptestctx.internal_dns.dropshot_server.local_addr() - ); - println!( - "omicron-dev: internal DNS: {}", - cptestctx.internal_dns.dns_server.local_address() - ); - println!( - "omicron-dev: external DNS name: {}", - cptestctx.external_dns_zone_name, - ); - println!( - "omicron-dev: external DNS HTTP: http://{}", - cptestctx.external_dns.dropshot_server.local_addr() - ); - println!( - "omicron-dev: external DNS: {}", - cptestctx.external_dns.dns_server.local_address() - ); - println!( - "omicron-dev: e.g. `dig @{} -p {} {}.sys.{}`", - cptestctx.external_dns.dns_server.local_address().ip(), - cptestctx.external_dns.dns_server.local_address().port(), - cptestctx.silo_name, - cptestctx.external_dns_zone_name, - ); - for (location, gateway) in &cptestctx.gateway { + let mut contexts = vec![]; + + for n in 0..self.count { + if self.count > 1 { + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_ip("0.0.0.0".parse().unwrap()); + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_port(0); + + config.deployment.dropshot_internal.bind_address.set_port(0); + config.deployment.dropshot_lockstep.bind_address.set_port(0); + config.deployment.techport_external_server_port = 0; + } else { + if let Some(p) = self.nexus_listen_port { + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_port(p); + } + } + + println!("\nomicron-dev: setting up all services for rack {n}... "); + let cptestctx = + nexus_test_utils::omicron_dev_setup_with_config::< + omicron_nexus::Server, + >(&mut config, 1, self.gateway_config.clone()) + .await + .context("error setting up services")?; + + println!("omicron-dev: Adding disks to first sled agent"); + + // This is how our integration tests are identifying that "disks exist" + // within the database. + // + // This inserts: + // - DEFAULT_ZPOOL_COUNT zpools, each of which contains: + // - A crucible dataset + // - A debug dataset + DiskTest::new(&cptestctx).await; + + println!("omicron-dev: services are running."); + + // Print out basic information about what was started. + // NOTE: The stdout strings here are not intended to be stable, but they + // are used by the test suite. + let addr = cptestctx.external_client.bind_address; + println!("omicron-dev: nexus external API: {:?}", addr); + println!( + "omicron-dev: nexus internal API: {:?}", + cptestctx.server.get_http_server_internal_address(), + ); + println!( + "omicron-dev: nexus lockstep API: {:?}", + cptestctx.server.get_http_server_lockstep_address(), + ); + println!( + "omicron-dev: cockroachdb pid: {}", + cptestctx.database.pid(), + ); + println!( + "omicron-dev: cockroachdb URL: {}", + cptestctx.database.pg_config() + ); + println!( + "omicron-dev: cockroachdb directory: {}", + cptestctx.database.temp_dir().display() + ); println!( - "omicron-dev: management gateway: {} ({})", - gateway.client.baseurl(), - location, + "omicron-dev: clickhouse native addr: {}", + cptestctx.clickhouse.native_address(), ); + println!( + "omicron-dev: clickhouse http addr: {}", + cptestctx.clickhouse.http_address(), + ); + println!( + "omicron-dev: internal DNS HTTP: http://{}", + cptestctx.internal_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: internal DNS: {}", + cptestctx.internal_dns.dns_server.local_address() + ); + println!( + "omicron-dev: external DNS name: {}", + cptestctx.external_dns_zone_name, + ); + println!( + "omicron-dev: external DNS HTTP: http://{}", + cptestctx.external_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: external DNS: {}", + cptestctx.external_dns.dns_server.local_address() + ); + println!( + "omicron-dev: e.g. `dig @{} -p {} {}.sys.{}`", + cptestctx.external_dns.dns_server.local_address().ip(), + cptestctx.external_dns.dns_server.local_address().port(), + cptestctx.silo_name, + cptestctx.external_dns_zone_name, + ); + for (location, gateway) in &cptestctx.gateway { + println!( + "omicron-dev: management gateway: {} ({})", + gateway.client.baseurl(), + location, + ); + } + for (location, dendrite) in + cptestctx.dendrite.read().unwrap().iter() + { + println!( + "omicron-dev: dendrite: http://[::1]:{} ({})", + dendrite.port, location, + ); + } + for (location, mgd) in &cptestctx.mgd { + println!( + "omicron-dev: maghemite: http://[::1]:{} ({})", + mgd.port, location, + ); + } + println!( + "omicron-dev: silo name: {}", + cptestctx.silo_name, + ); + println!( + "omicron-dev: privileged user name: {}", + cptestctx.user_name.as_ref(), + ); + println!( + "omicron-dev: privileged password: {}", + cptestctx.password + ); + contexts.push(cptestctx); } - println!( - "omicron-dev: silo name: {}", - cptestctx.silo_name, - ); - println!( - "omicron-dev: privileged user name: {}", - cptestctx.user_name.as_ref(), - ); - println!("omicron-dev: privileged password: {}", cptestctx.password); // Wait for a signal. let caught_signal = signal_stream.next().await; assert_eq!(caught_signal.unwrap(), SIGINT); eprintln!( "omicron-dev: caught signal, shutting down and removing \ - temporary directory" + temporary directory" ); - cptestctx.teardown().await; + for context in contexts { + context.teardown().await; + } + Ok(()) } } From 5a46d9952c2c3814417bfe164f2c8c845d713d1f Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Tue, 16 Dec 2025 01:34:58 +0000 Subject: [PATCH 2/9] WIP: integrate mgd changes --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- dev-tools/ls-apis/tests/api_dependencies.out | 1 + package-manifest.toml | 12 ++++++------ tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 4 ++-- 7 files changed, 17 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3d52c52bb4..544d3e8fa35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2467,7 +2467,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=0df320d42b356e689a3c7a7600eec9b16770237a#0df320d42b356e689a3c7a7600eec9b16770237a" +source = "git+https://github.com/oxidecomputer/maghemite?rev=48bc9972bbd2df6e61e4bbe875a58b6bc9075182#48bc9972bbd2df6e61e4bbe875a58b6bc9075182" dependencies = [ "oxnet", "progenitor 0.11.2", @@ -6312,7 +6312,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=0df320d42b356e689a3c7a7600eec9b16770237a#0df320d42b356e689a3c7a7600eec9b16770237a" +source = "git+https://github.com/oxidecomputer/maghemite?rev=48bc9972bbd2df6e61e4bbe875a58b6bc9075182#48bc9972bbd2df6e61e4bbe875a58b6bc9075182" dependencies = [ "chrono", "colored 3.0.0", @@ -11319,7 +11319,7 @@ dependencies = [ [[package]] name = "rdb-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=0df320d42b356e689a3c7a7600eec9b16770237a#0df320d42b356e689a3c7a7600eec9b16770237a" +source = "git+https://github.com/oxidecomputer/maghemite?rev=48bc9972bbd2df6e61e4bbe875a58b6bc9075182#48bc9972bbd2df6e61e4bbe875a58b6bc9075182" dependencies = [ "oxnet", "schemars 0.8.22", diff --git a/Cargo.toml b/Cargo.toml index 7a7a3148e2e..9cae37cec2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -538,8 +538,8 @@ newtype_derive = "0.1.6" ntp-admin-api = { path = "ntp-admin/api" } ntp-admin-client = { path = "clients/ntp-admin-client" } ntp-admin-types = { path = "ntp-admin/types" } -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "0df320d42b356e689a3c7a7600eec9b16770237a" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "0df320d42b356e689a3c7a7600eec9b16770237a" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } @@ -666,7 +666,7 @@ ratatui = "0.29.0" raw-cpuid = { git = "https://github.com/oxidecomputer/rust-cpuid.git", rev = "a4cf01df76f35430ff5d39dc2fe470bcb953503b" } rayon = "1.10" rcgen = "0.12.1" -rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "0df320d42b356e689a3c7a7600eec9b16770237a" } +rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" } reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" } reedline = "0.40.0" ref-cast = "1.0" diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out index 94197641c23..d48fe9fcfcf 100644 --- a/dev-tools/ls-apis/tests/api_dependencies.out +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -46,6 +46,7 @@ Downstairs Controller (debugging only) (client: dsc-client) Management Gateway Service (client: gateway-client) consumed by: dpd (dendrite/dpd) via 1 path + consumed by: mgd (maghemite/mgd) via 1 path consumed by: omicron-nexus (omicron/nexus) via 4 paths consumed by: omicron-sled-agent (omicron/sled-agent) via 1 path consumed by: wicketd (omicron/wicketd) via 3 paths diff --git a/package-manifest.toml b/package-manifest.toml index cd09595db18..f724897c0f0 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -654,10 +654,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "0df320d42b356e689a3c7a7600eec9b16770237a" +source.commit = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "0c6b756f317decd857b1641e300ed38c0f4cdfad4b77ee2322b2c8365fe95274" +source.sha256 = "eb7e25a37d0cff663ea947c5e7429e7074cde8e9228cecc2b63be7c0b056a87c" output.type = "tarball" [package.mg-ddm] @@ -670,10 +670,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "0df320d42b356e689a3c7a7600eec9b16770237a" +source.commit = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "5ef5389215ca3c1aaeef19a4abb0a01d85a6dabb0c2d6f02449883651e937fef" +source.sha256 = "ff9a875a6e354293de3e6ec08fa1da597fee768a9548ec95df3d53344974e7e6" output.type = "zone" output.intermediate_only = true @@ -685,10 +685,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "0df320d42b356e689a3c7a7600eec9b16770237a" +source.commit = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "e32f8202e69f7238efde2470f2f62e533fd78355a28e4ae99664fba3d91e8ae4" +source.sha256 = "adbf955030d7233764d425baad4e319a16e530cafacf1360e93ef4b6e61ab266" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index c066a90d021..336fcefd7e0 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1 +1 @@ -COMMIT="0df320d42b356e689a3c7a7600eec9b16770237a" +COMMIT="48bc9972bbd2df6e61e4bbe875a58b6bc9075182" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index c066a90d021..336fcefd7e0 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1 +1 @@ -COMMIT="0df320d42b356e689a3c7a7600eec9b16770237a" +COMMIT="48bc9972bbd2df6e61e4bbe875a58b6bc9075182" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 443275a8b92..96e2accfa6d 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="e32f8202e69f7238efde2470f2f62e533fd78355a28e4ae99664fba3d91e8ae4" -MGD_LINUX_SHA256="4e7dd03fd7c8fa33b658583c3e95c1b814001e8078e415a4a0dd28e98c87439e" \ No newline at end of file +CIDL_SHA256="adbf955030d7233764d425baad4e319a16e530cafacf1360e93ef4b6e61ab266" +MGD_LINUX_SHA256="9a228ac663d1e2b406d4e681424bab76b93119597a131a0b8d938151135a4575" From e135d806cf8a52e95bf8d16e53f0dce7e5ead923 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Wed, 17 Dec 2025 00:48:27 +0000 Subject: [PATCH 3/9] make mgd work correctly in omicron-dev (and test ctx) --- Cargo.lock | 6 +- Cargo.toml | 6 +- internal-dns/types/src/names.rs | 3 - nexus/reconfigurator/planning/src/example.rs | 1 - nexus/src/app/background/tasks/bfd.rs | 38 ++++---- nexus/src/app/background/tasks/networking.rs | 40 +-------- .../tasks/sync_switch_configuration.rs | 21 +++-- nexus/src/app/bfd.rs | 2 +- nexus/src/app/bgp.rs | 16 ++-- nexus/src/app/mod.rs | 87 +++++++++++++++---- nexus/test-utils/src/starter.rs | 7 +- package-manifest.toml | 12 +-- test-utils/src/dev/maghemite.rs | 13 ++- tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 4 +- 16 files changed, 146 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 544d3e8fa35..6f27c725cdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2467,7 +2467,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=48bc9972bbd2df6e61e4bbe875a58b6bc9075182#48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +source = "git+https://github.com/oxidecomputer/maghemite?rev=9e94d6b79560c2e4639cba432fb0ed600e9a3ff8#9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" dependencies = [ "oxnet", "progenitor 0.11.2", @@ -6312,7 +6312,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=48bc9972bbd2df6e61e4bbe875a58b6bc9075182#48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +source = "git+https://github.com/oxidecomputer/maghemite?rev=9e94d6b79560c2e4639cba432fb0ed600e9a3ff8#9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" dependencies = [ "chrono", "colored 3.0.0", @@ -11319,7 +11319,7 @@ dependencies = [ [[package]] name = "rdb-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=48bc9972bbd2df6e61e4bbe875a58b6bc9075182#48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +source = "git+https://github.com/oxidecomputer/maghemite?rev=9e94d6b79560c2e4639cba432fb0ed600e9a3ff8#9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" dependencies = [ "oxnet", "schemars 0.8.22", diff --git a/Cargo.toml b/Cargo.toml index 9cae37cec2a..f5b2e665779 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -538,8 +538,8 @@ newtype_derive = "0.1.6" ntp-admin-api = { path = "ntp-admin/api" } ntp-admin-client = { path = "clients/ntp-admin-client" } ntp-admin-types = { path = "ntp-admin/types" } -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } @@ -666,7 +666,7 @@ ratatui = "0.29.0" raw-cpuid = { git = "https://github.com/oxidecomputer/rust-cpuid.git", rev = "a4cf01df76f35430ff5d39dc2fe470bcb953503b" } rayon = "1.10" rcgen = "0.12.1" -rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" } +rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" } reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" } reedline = "0.40.0" ref-cast = "1.0" diff --git a/internal-dns/types/src/names.rs b/internal-dns/types/src/names.rs index 323b2aea07f..fb150c646e1 100644 --- a/internal-dns/types/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -76,7 +76,6 @@ pub enum ServiceName { Crucible(OmicronZoneUuid), BoundaryNtp, InternalNtp, - Maghemite, //TODO change to Dpd - maghemite has several services. Mgd, } @@ -120,7 +119,6 @@ impl ServiceName { ServiceName::Crucible(_) => "crucible", ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", - ServiceName::Maghemite => "maghemite", ServiceName::Mgd => "mgd", } } @@ -152,7 +150,6 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp - | ServiceName::Maghemite | ServiceName::Mgd => { format!("_{}._tcp", self.service_kind()) } diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index efd690b2815..bcf48f2c1ed 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -1424,7 +1424,6 @@ mod tests { | ServiceName::Dendrite | ServiceName::Tfport | ServiceName::BoundaryNtp - | ServiceName::Maghemite | ServiceName::Mgd => { out.insert(service, Err(QueryError::NoRecordsFound)); } diff --git a/nexus/src/app/background/tasks/bfd.rs b/nexus/src/app/background/tasks/bfd.rs index 8662124e25a..8d3b1c3c4e8 100644 --- a/nexus/src/app/background/tasks/bfd.rs +++ b/nexus/src/app/background/tasks/bfd.rs @@ -5,12 +5,8 @@ //! Background task for managing switch bidirectional forwarding detection //! (BFD) sessions. -use crate::app::{ - background::tasks::networking::build_mgd_clients, - switch_zone_address_mappings, -}; - use crate::app::background::BackgroundTask; +use crate::app::mgd_clients; use futures::FutureExt; use futures::future::BoxFuture; use internal_dns_resolver::Resolver; @@ -118,21 +114,23 @@ impl BackgroundTask for BfdManager { let mut current: HashSet = HashSet::new(); - let mappings = match switch_zone_address_mappings(&self.resolver, log).await { - Ok(mappings) => mappings, - Err(e) => { - error!(log, "failed to resolve addresses for Dendrite services"; "error" => %e); - return json!({ - "error": - format!( - "failed to resolve addresses for Dendrite services: {:#}", - e - ) - }); - }, - }; - - let mgd_clients = build_mgd_clients(mappings, log, &self.resolver).await; + let mgd_clients = match mgd_clients(&self.resolver, log).await + { + Ok(mappings) => mappings, + Err(e) => { + error!( + log, + "failed to resolve addresses for Maghemite"; + "error" => %e); + return json!({ + "error": + format!( + "failed to resolve addresses for Maghemite: {:#}", + e + ) + }); + }, + }; for (location, c) in &mgd_clients { let client_current = match c.get_bfd_peers().await { diff --git a/nexus/src/app/background/tasks/networking.rs b/nexus/src/app/background/tasks/networking.rs index 7ff50a5798d..767a870fb1c 100644 --- a/nexus/src/app/background/tasks/networking.rs +++ b/nexus/src/app/background/tasks/networking.rs @@ -6,47 +6,9 @@ use db::datastore::SwitchPortSettingsCombinedResult; use dpd_client::types::{ LinkCreate, LinkId, LinkSettings, PortFec, PortSettings, PortSpeed, TxEq, }; -use internal_dns_types::names::ServiceName; use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; use nexus_db_queries::db; -use omicron_common::{address::MGD_PORT, api::external::SwitchLocation}; -use std::{ - collections::HashMap, - net::{Ipv6Addr, SocketAddrV6}, -}; - -pub(crate) async fn build_mgd_clients( - mappings: HashMap, - log: &slog::Logger, - resolver: &internal_dns_resolver::Resolver, -) -> HashMap { - let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> = vec![]; - for (location, addr) in &mappings { - let port = match resolver.lookup_all_socket_v6(ServiceName::Mgd).await { - Ok(addrs) => { - let port_map: HashMap = addrs - .into_iter() - .map(|sockaddr| (*sockaddr.ip(), sockaddr.port())) - .collect(); - - *port_map.get(&addr).unwrap_or(&MGD_PORT) - } - Err(e) => { - error!(log, "failed to addresses"; "error" => %e); - MGD_PORT - } - }; - - let socketaddr = - std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0)); - let client = mg_admin_client::Client::new( - format!("http://{}", socketaddr).as_str(), - log.clone(), - ); - clients.push((*location, client)); - } - clients.into_iter().collect::>() -} +use std::collections::HashMap; pub(crate) fn api_to_dpd_port_settings( settings: &SwitchPortSettingsCombinedResult, diff --git a/nexus/src/app/background/tasks/sync_switch_configuration.rs b/nexus/src/app/background/tasks/sync_switch_configuration.rs index 9a91dd28a7c..a10c33d87dc 100644 --- a/nexus/src/app/background/tasks/sync_switch_configuration.rs +++ b/nexus/src/app/background/tasks/sync_switch_configuration.rs @@ -6,10 +6,8 @@ //! to relevant management daemons (dendrite, mgd, sled-agent, etc.) use crate::app::{ - background::tasks::networking::{ - api_to_dpd_port_settings, build_mgd_clients, - }, - dpd_clients, switch_zone_address_mappings, + background::tasks::networking::api_to_dpd_port_settings, dpd_clients, + mgd_clients, switch_zone_address_mappings, }; use oxnet::Ipv4Net; use slog::{Logger, o}; @@ -393,8 +391,19 @@ impl BackgroundTask for SwitchPortSettingsManager { }; // TODO https://github.com/oxidecomputer/omicron/issues/5201 - // build mgd clients - let mgd_clients = build_mgd_clients(mappings, &log, &self.resolver).await; + let mgd_clients = match + mgd_clients(&self.resolver, &log).await + { + Ok(mappings) => mappings, + Err(e) => { + error!( + log, + "failed to resolve addresses for Maghemite"; + "error" => %e); + continue; + }, + }; + let port_list = match self.switch_ports(opctx, &log).await { Ok(value) => value, diff --git a/nexus/src/app/bfd.rs b/nexus/src/app/bfd.rs index 1ae958c20d4..026c85a85cb 100644 --- a/nexus/src/app/bfd.rs +++ b/nexus/src/app/bfd.rs @@ -14,7 +14,7 @@ impl super::Nexus { switch: SwitchLocation, ) -> Result { let mg_client: mg_admin_client::Client = self - .mg_clients() + .mgd_clients() .await .map_err(|e| { Error::internal_error(&format!("failed to get mg clients: {e}")) diff --git a/nexus/src/app/bgp.rs b/nexus/src/app/bgp.rs index 60f23d958d4..94e5f10dba8 100644 --- a/nexus/src/app/bgp.rs +++ b/nexus/src/app/bgp.rs @@ -104,9 +104,9 @@ impl super::Nexus { ) -> ListResultVec { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = Vec::new(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let router_info = match client.read_routers().await { @@ -161,9 +161,9 @@ impl super::Nexus { ) -> LookupResult { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = BgpExported::default(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let router_info = match client.read_routers().await { @@ -231,9 +231,9 @@ impl super::Nexus { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = Vec::new(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let history = match client @@ -273,9 +273,9 @@ impl super::Nexus { ) -> ListResultVec { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; let mut result = Vec::new(); - for (switch, client) in &self.mg_clients().await.map_err(|e| { + for (switch, client) in &self.mgd_clients().await.map_err(|e| { external::Error::internal_error(&format!( - "failed to get mg clients: {e}" + "failed to get mgd clients: {e}" )) })? { let mut imported: Vec = Vec::new(); diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index f9a8057958c..90cc7a506a3 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -31,7 +31,6 @@ use nexus_mgs_updates::MgsUpdateDriver; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::ReconfiguratorConfigParam; use nexus_types::fm; -use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Error; @@ -1123,25 +1122,11 @@ impl Nexus { lldpd_clients(resolver, rack_id, &self.log).await } - pub(crate) async fn mg_clients( + pub(crate) async fn mgd_clients( &self, ) -> Result, String> { let resolver = self.resolver(); - let mappings = - switch_zone_address_mappings(resolver, &self.log).await?; - let mut clients: Vec<(SwitchLocation, mg_admin_client::Client)> = - vec![]; - for (location, addr) in &mappings { - let port = MGD_PORT; - let socketaddr = - std::net::SocketAddr::V6(SocketAddrV6::new(*addr, port, 0, 0)); - let client = mg_admin_client::Client::new( - format!("http://{}", socketaddr).as_str(), - self.log.clone(), - ); - clients.push((*location, client)); - } - Ok(clients.into_iter().collect::>()) + mgd_clients(resolver, &self.log).await } pub(crate) fn demo_sagas( @@ -1269,6 +1254,74 @@ pub(crate) async fn dpd_clients( Ok(mappings) } +/// Returns a mapping of clients for the Maghemite daemons of reachable switch zones. +/// If we are unable to communicate with the switch zone and determine the mapping +/// of SwitchLocation -> Zone Underlay Address, we omit an entry for that client. +pub(crate) async fn mgd_clients( + resolver: &internal_dns_resolver::Resolver, + log: &slog::Logger, +) -> Result, String> { + let mgd_socketaddrs = match resolver + .lookup_all_socket_v6(ServiceName::Mgd) + .await + { + Ok(addrs) => addrs, + Err(e) => { + error!(log, "failed to resolve addresses for Maghemite services"; "error" => %e); + return Err(e.to_string()); + } + }; + + let clients: Vec<(SocketAddrV6, mg_admin_client::Client)> = mgd_socketaddrs + .iter() + .map(|socket_addr| { + let client = mg_admin_client::Client::new( + &format!("http://{socket_addr}"), + log.new(o!( + "component" => "MgdClient" + )), + ); + + (*socket_addr, client) + }) + .collect(); + + let mut mappings: HashMap = + HashMap::new(); + + for (addr, client) in clients { + let switch_slot = match client.switch_identifiers().await { + Ok(response) => response.slot, + Err(e) => { + error!( + log, + "failed to determine switch slot for maghemite"; + "error" => %e, + "addr" => %addr, + ); + continue; + } + }; + + let location = match switch_slot { + Some(0) => SwitchLocation::Switch0, + Some(1) => SwitchLocation::Switch1, + Some(v) => { + warn!(log, "unexpected value for switch slot: {v}"); + continue; + } + None => { + warn!(log, "maghemite has not learned switch slot from MGS"); + continue; + } + }; + + mappings.insert(location, client); + } + + Ok(mappings) +} + // We currently ignore the rack_id argument here, as the shared // switch_zone_address_mappings function doesn't allow filtering on the rack ID. // Since we only have a single rack, this is OK for now. diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 412959d3d63..968a3c4a7df 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -444,9 +444,14 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { pub async fn start_mgd(&mut self, switch_location: SwitchLocation) { let log = &self.logctx.log; debug!(log, "Starting mgd for {switch_location}"); + let mgs = self.gateway.get(&switch_location).unwrap(); + let mgs_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); // Set up an instance of mgd - let mgd = dev::maghemite::MgdInstance::start(0).await.unwrap(); + let mgd = dev::maghemite::MgdInstance::start(0, Some(mgs_addr)) + .await + .unwrap(); let port = mgd.port; self.mgd.insert(switch_location, mgd); let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); diff --git a/package-manifest.toml b/package-manifest.toml index f724897c0f0..dc7307af99c 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -654,10 +654,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +source.commit = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "eb7e25a37d0cff663ea947c5e7429e7074cde8e9228cecc2b63be7c0b056a87c" +source.sha256 = "4a8068799336e59dd1baf93e83c66c22b9fab7375950dbfddae55596eb9448bd" output.type = "tarball" [package.mg-ddm] @@ -670,10 +670,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +source.commit = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "ff9a875a6e354293de3e6ec08fa1da597fee768a9548ec95df3d53344974e7e6" +source.sha256 = "8f01f601b462314d309b01d3f13672b3f24966c4a2266213506543dcc1fc9644" output.type = "zone" output.intermediate_only = true @@ -685,10 +685,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +source.commit = "9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "adbf955030d7233764d425baad4e319a16e530cafacf1360e93ef4b6e61ab266" +source.sha256 = "b9b7eb4fc04b539c432a5cbc978b53f77ad6942a17137ac2912971e178a68917" output.type = "zone" output.intermediate_only = true diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index 4c2d85df3ee..343b67c08aa 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -4,6 +4,7 @@ //! Tools for managing Maghemite during development +use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; @@ -35,10 +36,13 @@ pub struct MgdInstance { } impl MgdInstance { - pub async fn start(mut port: u16) -> Result { + pub async fn start( + mut port: u16, + mgs_address: Option, + ) -> Result { let temp_dir = TempDir::new()?; - let args = vec![ + let mut args = vec![ "run".to_string(), "--admin-addr".into(), "::1".into(), @@ -53,6 +57,11 @@ impl MgdInstance { uuid::Uuid::new_v4().to_string(), ]; + if let Some(socket_addr) = mgs_address { + args.push("--mgs-addr".to_string()); + args.push(socket_addr.to_string()); + } + let child = tokio::process::Command::new("mgd") .args(&args) .stdin(Stdio::null()) diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 336fcefd7e0..c4bcd2c981e 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1 +1 @@ -COMMIT="48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +COMMIT="4a096d2fba90f46ce7c59318e0f4bb9b9d76ed20" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 336fcefd7e0..bdd090ee327 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1 +1 @@ -COMMIT="48bc9972bbd2df6e61e4bbe875a58b6bc9075182" +COMMIT="9e94d6b79560c2e4639cba432fb0ed600e9a3ff8" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 96e2accfa6d..47f1a2f3509 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="adbf955030d7233764d425baad4e319a16e530cafacf1360e93ef4b6e61ab266" -MGD_LINUX_SHA256="9a228ac663d1e2b406d4e681424bab76b93119597a131a0b8d938151135a4575" +CIDL_SHA256="b9b7eb4fc04b539c432a5cbc978b53f77ad6942a17137ac2912971e178a68917" +MGD_LINUX_SHA256="0f2bbd3cb8dc061242f2a7735660fff483918c3fb928b0737c688420c78430ba" From 1600e1c122651b8903ecacbb5acb8dc5789ec240 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Wed, 17 Dec 2025 20:41:09 +0000 Subject: [PATCH 4/9] update test fixture --- dev-tools/omdb/tests/successes.out | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 2738d6ff1fb..f915570374f 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -573,7 +573,7 @@ task: "bfd_manager" configured period: every s last completed activation: , triggered by started at (s ago) and ran for ms - last completion reported error: failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } + last completion reported error: failed to resolve addresses for Maghemite: proto error: no records found for Query { name: Name("_mgd._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "blueprint_planner" configured period: every m @@ -1141,7 +1141,7 @@ task: "bfd_manager" configured period: every s last completed activation: , triggered by started at (s ago) and ran for ms - last completion reported error: failed to resolve addresses for Dendrite services: proto error: no records found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } + last completion reported error: failed to resolve addresses for Maghemite: proto error: no records found for Query { name: Name("_mgd._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "blueprint_planner" configured period: every m From b3dcdf452d427947bfe8c31f6d2839f9b31b2ca6 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Tue, 23 Dec 2025 02:17:37 +0000 Subject: [PATCH 5/9] WIP: add LLDPD --- .envrc | 1 + dev-tools/downloader/src/lib.rs | 80 +++++++++++++++++++ dev-tools/omicron-dev/src/main.rs | 6 ++ env.sh | 1 + internal-dns/types/src/config.rs | 4 +- internal-dns/types/src/names.rs | 3 + nexus-config/src/nexus_config.rs | 18 ++++- nexus/test-utils/src/nexus_test.rs | 1 + nexus/test-utils/src/starter.rs | 45 +++++++++++ nexus/tests/integration_tests/schema.rs | 2 + nexus/types/src/deployment/execution/dns.rs | 1 + .../src/deployment/execution/overridables.rs | 13 +++ package-manifest.toml | 4 +- sled-agent/src/rack_setup/plan/service.rs | 2 + test-utils/src/dev/mod.rs | 1 + tools/install_builder_prerequisites.sh | 3 + tools/lldp_checksums | 2 + tools/lldp_openapi_version | 1 + 18 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 tools/lldp_checksums create mode 100644 tools/lldp_openapi_version diff --git a/.envrc b/.envrc index 48df8e3c630..bb15c030860 100644 --- a/.envrc +++ b/.envrc @@ -5,6 +5,7 @@ PATH_add out/cockroachdb/bin PATH_add out/clickhouse PATH_add out/dendrite-stub/bin PATH_add out/mgd/root/opt/oxide/mgd/bin +PATH_add out/lldp/root/opt/oxide/bin if [ "$OMICRON_USE_FLAKE" = 1 ] && nix flake show &> /dev/null then diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index 44fb340de28..bd718dfa810 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -74,6 +74,9 @@ enum Target { /// Transceiver Control binary TransceiverControl, + + /// LLDP binary + LLDP, } #[derive(Parser)] @@ -136,6 +139,7 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> { Target::Cockroach => downloader.download_cockroach().await, Target::Console => downloader.download_console().await, Target::DendriteStub => downloader.download_dendrite_stub().await, + Target::LLDP => downloader.download_lldp().await, Target::MaghemiteMgd => downloader.download_maghemite_mgd().await, Target::Softnpu => downloader.download_softnpu().await, Target::TransceiverControl => { @@ -870,6 +874,82 @@ impl Downloader<'_> { Ok(()) } + async fn download_lldp(&self) -> std::result::Result<(), anyhow::Error> { + let download_dir = self.output_dir.join("downloads"); + tokio::fs::create_dir_all(&download_dir).await?; + + let checksums_path = self.versions_dir.join("lldp_checksums"); + let [lldp_sha2, lldp_linux_sha2] = get_values_from_file( + ["CIDL_SHA256", "LINUX_SHA256"], + &checksums_path, + ) + .await?; + let commit_path = + self.versions_dir.join("lldp_openapi_version"); + let [commit] = get_values_from_file(["COMMIT"], &commit_path).await?; + + let repo = "oxidecomputer/lldp"; + let base_url = format!("{BUILDOMAT_URL}/{repo}/image/{commit}"); + + let filename = "lldp.tar.gz"; + let tarball_path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &tarball_path, + &format!("{base_url}/{filename}"), + ChecksumAlgorithm::Sha2, + &lldp_sha2, + ) + .await?; + unpack_tarball(&self.log, &tarball_path, &download_dir).await?; + + let destination_dir = self.output_dir.join("lldp"); + let _ = tokio::fs::remove_dir_all(&destination_dir).await; + tokio::fs::create_dir_all(&destination_dir).await?; + copy_dir_all( + &download_dir.join("root"), + &destination_dir.join("root"), + )?; + + let binary_dir = destination_dir.join("root/opt/oxide/lldp/bin"); + + match os_name()? { + Os::Linux => { + let filename = "lldp"; + let path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &path, + &format!( + "{BUILDOMAT_URL}/{repo}/linux/{commit}/{filename}" + ), + ChecksumAlgorithm::Sha2, + &lldp_linux_sha2, + ) + .await?; + set_permissions(&path, 0o755).await?; + tokio::fs::copy(path, binary_dir.join(filename)).await?; + } + Os::Mac => { + info!(self.log, "Building lldp from source for macOS"); + + let binaries = [("lldp", &["--no-default-features"][..])]; + + let built_binaries = self + .build_from_git("lldp", &commit, &binaries) + .await?; + + // Copy built binary to binary_dir + let dest = binary_dir.join("lldp"); + tokio::fs::copy(&built_binaries[0], &dest).await?; + set_permissions(&dest, 0o755).await?; + } + Os::Illumos => (), + } + + Ok(()) + } + async fn download_maghemite_mgd(&self) -> Result<()> { let download_dir = self.output_dir.join("downloads"); tokio::fs::create_dir_all(&download_dir).await?; diff --git a/dev-tools/omicron-dev/src/main.rs b/dev-tools/omicron-dev/src/main.rs index 8e027c27afa..2d037f4f2bf 100644 --- a/dev-tools/omicron-dev/src/main.rs +++ b/dev-tools/omicron-dev/src/main.rs @@ -208,6 +208,12 @@ impl RunAllArgs { dendrite.port, location, ); } + for (location, lldpd) in &cptestctx.lldpd { + println!( + "omicron-dev: lldp: http://[::1]:{} ({})", + lldpd.port, location, + ); + } for (location, mgd) in &cptestctx.mgd { println!( "omicron-dev: maghemite: http://[::1]:{} ({})", diff --git a/env.sh b/env.sh index 6a84c35902a..1ced95c37bb 100644 --- a/env.sh +++ b/env.sh @@ -12,6 +12,7 @@ export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" +export PATH="$OMICRON_WS/out/lldp/root/opt/oxide/bin:$PATH" # if xtrace was set previously, do not unset it case $OLD_SHELL_OPTS in diff --git a/internal-dns/types/src/config.rs b/internal-dns/types/src/config.rs index 961d4fac554..01d6811865e 100644 --- a/internal-dns/types/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -399,6 +399,7 @@ impl DnsConfigBuilder { dendrite_port: u16, mgs_port: u16, mgd_port: u16, + lldpd_port: u16, ) -> anyhow::Result<()> { let zone = self.host_dendrite(sled_id, switch_zone_ip)?; self.service_backend_zone(ServiceName::Dendrite, &zone, dendrite_port)?; @@ -407,7 +408,8 @@ impl DnsConfigBuilder { &zone, mgs_port, )?; - self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port) + self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port)?; + self.service_backend_zone(ServiceName::Lldpd, &zone, lldpd_port) } /// Higher-level shorthand for adding a Nexus zone with both its internal diff --git a/internal-dns/types/src/names.rs b/internal-dns/types/src/names.rs index fb150c646e1..eb4c77a743c 100644 --- a/internal-dns/types/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -77,6 +77,7 @@ pub enum ServiceName { BoundaryNtp, InternalNtp, Mgd, + Lldpd, } impl ServiceName { @@ -120,6 +121,7 @@ impl ServiceName { ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", ServiceName::Mgd => "mgd", + ServiceName::Lldpd => "lldpd", } } @@ -150,6 +152,7 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp + | ServiceName::Lldpd | ServiceName::Mgd => { format!("_{}._tcp", self.service_kind()) } diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 4885362ed36..71c8c0350df 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -268,7 +268,13 @@ pub struct DpdConfig { pub address: SocketAddr, } -/// Configuration for the `Dendrite` dataplane daemon. +/// Configuration for the `LLDP` daemon. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub struct LldpdConfig { + pub address: SocketAddr, +} + +/// Configuration for the `Maghemite` routing daemon. #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct MgdConfig { pub address: SocketAddr, @@ -1003,6 +1009,9 @@ pub struct PackageConfig { /// `Dendrite` dataplane daemon configuration #[serde(default)] pub dendrite: HashMap, + /// `LLDP` daemon configuration + #[serde(default)] + pub lldpd: HashMap, /// Maghemite mgd daemon configuration #[serde(default)] pub mgd: HashMap, @@ -1370,6 +1379,13 @@ mod test { .unwrap(), } )]), + lldpd: HashMap::from([( + SwitchLocation::Switch0, + LldpdConfig { + address: SocketAddr::from_str("[::1]:12230") + .unwrap(), + } + )]), mgd: HashMap::from([( SwitchLocation::Switch0, MgdConfig { diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index cd5c2f694a2..d51c5ee8f3d 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -115,6 +115,7 @@ pub struct ControlPlaneTestContext { pub gateway: BTreeMap, pub dendrite: RwLock>, + pub lldpd: HashMap, pub mgd: HashMap, pub external_dns_zone_name: String, pub external_dns: dns_server::TransientServer, diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 968a3c4a7df..3ca2d3e4e2e 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -25,6 +25,7 @@ use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use internal_dns_types::names::ServiceName; use nexus_config::Database; use nexus_config::DpdConfig; +use nexus_config::LldpdConfig; use nexus_config::InternalDns; use nexus_config::MgdConfig; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; @@ -142,6 +143,7 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub dendrite: RwLock>, pub mgd: HashMap, + pub lldpd: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -198,6 +200,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { producer: None, gateway: BTreeMap::new(), dendrite: RwLock::new(HashMap::new()), + lldpd: HashMap::new(), mgd: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, @@ -441,6 +444,34 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.config.pkg.dendrite.insert(switch_location, config); } + pub async fn start_lldp(&mut self, switch_location: SwitchLocation) { + let log = &self.logctx.log; + debug!(log, "Starting LLDP for {switch_location}"); + let mgs = self.gateway.get(&switch_location).unwrap(); + let mgs_addr = + SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); + + let dpd_port = self.dendrite.read().unwrap().get(&switch_location).unwrap().port; + + // Set up an instance of lldpd + let lldpd = dev::lldp::LldpdInstance::start( + 0, + dpd_port, + Some(mgs_addr), + ) + .await + .unwrap(); + + let port = lldpd.port; + self.lldpd.insert(switch_location, lldpd); + let address = SocketAddrV6::new(Ipv6Addr::LOCALHOST, port, 0, 0); + + debug!(log, "lldp port is {port}"); + + let config = LldpdConfig { address: std::net::SocketAddr::V6(address) }; + self.config.pkg.lldpd.insert(switch_location, config); + } + pub async fn start_mgd(&mut self, switch_location: SwitchLocation) { let log = &self.logctx.log; debug!(log, "Starting mgd for {switch_location}"); @@ -488,6 +519,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { .port, self.gateway.get(&switch_location).unwrap().port, self.mgd.get(&switch_location).unwrap().port, + self.lldpd.get(&switch_location).unwrap().port, ) .unwrap() } @@ -1259,6 +1291,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { logctx: self.logctx, gateway: self.gateway, dendrite: RwLock::new(self.dendrite.into_inner().unwrap()), + lldpd: self.lldpd, mgd: self.mgd, external_dns_zone_name: self.external_dns_zone_name.unwrap(), external_dns: self.external_dns.unwrap(), @@ -1632,6 +1665,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_dendrite(SwitchLocation::Switch0).boxed() }), ), + ( + "start_lldpd_switch0", + Box::new(|builder| { + builder.start_lldp(SwitchLocation::Switch0).boxed() + }), + ), ( "start_mgd_switch0", Box::new(|builder| { @@ -1678,6 +1717,12 @@ pub(crate) async fn setup_with_config_impl( .boxed() }), ), + ( + "start_lldpd_switch1", + Box::new(|builder| { + builder.start_lldp(SwitchLocation::Switch1).boxed() + }), + ), ( "start_mgd_switch1", Box::new(|builder| { diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 8a50c863d1f..6042b3262ee 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -59,6 +59,8 @@ async fn test_setup<'a>( starter.start_gateway(SwitchLocation::Switch1, None, sp_conf).await; starter.start_dendrite(SwitchLocation::Switch0).await; starter.start_dendrite(SwitchLocation::Switch1).await; + starter.start_lldp(SwitchLocation::Switch0).await; + starter.start_lldp(SwitchLocation::Switch1).await; starter.start_mgd(SwitchLocation::Switch0).await; starter.start_mgd(SwitchLocation::Switch1).await; starter.populate_internal_dns().await; diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs index 2e509d12b28..3953adbef37 100644 --- a/nexus/types/src/deployment/execution/dns.rs +++ b/nexus/types/src/deployment/execution/dns.rs @@ -161,6 +161,7 @@ pub fn blueprint_internal_dns_config( overrides.dendrite_port(scrimlet.id()), overrides.mgs_port(scrimlet.id()), overrides.mgd_port(scrimlet.id()), + overrides.lldpd_port(scrimlet.id()), )?; } diff --git a/nexus/types/src/deployment/execution/overridables.rs b/nexus/types/src/deployment/execution/overridables.rs index 881a7c49bdd..154902d1a8c 100644 --- a/nexus/types/src/deployment/execution/overridables.rs +++ b/nexus/types/src/deployment/execution/overridables.rs @@ -4,6 +4,7 @@ use omicron_common::address::DENDRITE_PORT; use omicron_common::address::Ipv6Subnet; +use omicron_common::address::LLDP_PORT; use omicron_common::address::MGD_PORT; use omicron_common::address::MGS_PORT; use omicron_common::address::SLED_PREFIX; @@ -31,6 +32,8 @@ pub struct Overridables { pub mgd_ports: BTreeMap, /// map: sled id -> IP address of the sled's switch zone pub switch_zone_ips: BTreeMap, + /// map: sled id -> TCP port on which that sled's LLDP is listening + pub lldpd_ports: BTreeMap, } pub static DEFAULT: LazyLock = @@ -67,6 +70,16 @@ impl Overridables { self.mgd_ports.get(&sled_id).copied().unwrap_or(MGD_PORT) } + /// Specify the TCP port on which this sled's LLDP is listening + pub fn override_lldpd_port(&mut self, sled_id: SledUuid, port: u16) { + self.lldpd_ports.insert(sled_id, port); + } + + /// Returns the TCP port on which this sled's LLDP is listening + pub fn lldpd_port(&self, sled_id: SledUuid) -> u16 { + self.lldpd_ports.get(&sled_id).copied().unwrap_or(LLDP_PORT) + } + /// Specify the IP address of this switch zone pub fn override_switch_zone_ip( &mut self, diff --git a/package-manifest.toml b/package-manifest.toml index dc7307af99c..6e2054535fd 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -696,8 +696,8 @@ output.intermediate_only = true service_name = "lldp" source.type = "prebuilt" source.repo = "lldp" -source.commit = "61479b6922f9112fbe1e722414d2b8055212cb12" -source.sha256 = "8f988c0b0fa3ad4121ab0e825298601035e56c5c054bdc3a1dfb4d6c8fd5b300" +source.commit = "1f7743e5f7ce37a3315dd7d7f35dc4992c3716ff" +source.sha256 = "b7e25ae3e90392b2fd85150ffb830fe43c5cec56036b370e97160c924fa37d00" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index f511f1de056..60ee652b57a 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -26,6 +26,7 @@ use nexus_types::deployment::{ blueprint_zone_type, }; use nexus_types::external_api::views::SledState; +use omicron_common::address::LLDP_PORT; use omicron_common::address::{ DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, NEXUS_LOCKSTEP_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, @@ -333,6 +334,7 @@ impl Plan { DENDRITE_PORT, MGS_PORT, MGD_PORT, + LLDP_PORT, ) .unwrap(); } diff --git a/test-utils/src/dev/mod.rs b/test-utils/src/dev/mod.rs index 5a37ac2549e..6e573d03c54 100644 --- a/test-utils/src/dev/mod.rs +++ b/test-utils/src/dev/mod.rs @@ -9,6 +9,7 @@ pub mod clickhouse; pub mod db; pub mod dendrite; pub mod falcon; +pub mod lldp; pub mod maghemite; pub mod poll; #[cfg(feature = "seed-gen")] diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 646c764f8e9..40ad1fce2e2 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -223,6 +223,7 @@ retry xtask download \ console \ dendrite-stub \ maghemite-mgd \ + lldp \ transceiver-control # Validate the PATH: @@ -232,6 +233,8 @@ expected_in_path=( 'cockroach' 'clickhouse' 'dpd' + 'mgd' + 'lldpd' ) function show_hint diff --git a/tools/lldp_checksums b/tools/lldp_checksums new file mode 100644 index 00000000000..4cc15667caa --- /dev/null +++ b/tools/lldp_checksums @@ -0,0 +1,2 @@ +CIDL_SHA256="b7e25ae3e90392b2fd85150ffb830fe43c5cec56036b370e97160c924fa37d00" +LINUX_SHA256="b51b5bcf265c788784a31ab6b12d2e35915505f5d7e6f4a3cba329e424d3ba81" diff --git a/tools/lldp_openapi_version b/tools/lldp_openapi_version new file mode 100644 index 00000000000..3b977fe3ece --- /dev/null +++ b/tools/lldp_openapi_version @@ -0,0 +1 @@ +COMMIT="1f7743e5f7ce37a3315dd7d7f35dc4992c3716ff" From bebb200a9da0c7bd46932b7d9fd50d96f07a00ee Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Tue, 23 Dec 2025 02:43:45 +0000 Subject: [PATCH 6/9] kinda need this file :P --- test-utils/src/dev/lldp.rs | 209 +++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 test-utils/src/dev/lldp.rs diff --git a/test-utils/src/dev/lldp.rs b/test-utils/src/dev/lldp.rs new file mode 100644 index 00000000000..808927d6b6f --- /dev/null +++ b/test-utils/src/dev/lldp.rs @@ -0,0 +1,209 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tools for managing LLDP during development + +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::Context; +use tempfile::TempDir; +use tokio::{ + fs::File, + io::{AsyncBufReadExt, BufReader}, + time::{Instant, sleep}, +}; + +/// Specifies the amount of time we will wait for `lldpd` to launch, +/// which is currently confirmed by watching `lldpd`'s log output +/// for a message specifying the address and port `lldpd` is listening on. +pub const LLDPD_TIMEOUT: Duration = Duration::new(5, 0); + +pub struct LldpdInstance { + /// Port number the mgd instance is listening on. This can be provided + /// manually, or dynamically determined if a value of 0 is provided. + pub port: u16, + /// Arguments provided to the `lldpd` cli command. + pub args: Vec, + /// Child process spawned by running `lldpd` + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `lldpd` are stored. + pub data_dir: Option, +} + +impl LldpdInstance { + pub async fn start( + mut port: u16, + dpd_port: u16, + mgs_address: Option, + ) -> Result { + let temp_dir = TempDir::new()?; + let listen_addr = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port); + + let mut args = vec![ + "run".to_string(), + "--listen-addr".into(), + listen_addr.to_string(), + "--port".to_string(), + dpd_port.to_string(), + ]; + + if let Some(socket_addr) = mgs_address { + args.push("--mgs-addr".to_string()); + args.push(socket_addr.to_string()); + } + + let child = tokio::process::Command::new("lldpd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file(temp_dir.path(), "lldpd_stdout")?)) + .stderr(Stdio::from(redirect_file(temp_dir.path(), "lldpd_stderr")?)) + .spawn() + .with_context(|| { + format!("failed to spawn `lldpd` (with args: {:?})", &args) + })?; + + let child = Some(child); + + let temp_dir = temp_dir.keep(); + if port == 0 { + port = discover_port( + temp_dir.join("lldpd_stdout").display().to_string(), + ) + .await + .with_context(|| { + format!( + "failed to discover lldpd port from files in {}", + temp_dir.display() + ) + })?; + } + + Ok(Self { port, args, child, data_dir: Some(temp_dir) }) + } + + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; + } + Ok(()) + } +} + +impl Drop for LldpdInstance { + fn drop(&mut self) { + if self.child.is_some() || self.data_dir.is_some() { + eprintln!( + "WARN: dropped LldpdInstance without cleaning it up first \ + (there may still be a child process running and a \ + temporary directory leaked)" + ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: lldpd temporary directory leaked: {}", + path.display() + ); + } + } + } +} + +fn redirect_file( + temp_dir_path: &Path, + label: &str, +) -> Result { + let out_path = temp_dir_path.join(label); + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&out_path) + .with_context(|| format!("open \"{}\"", out_path.display())) +} + +async fn discover_port(logfile: String) -> Result { + let timeout = Instant::now() + LLDPD_TIMEOUT; + tokio::time::timeout_at(timeout, find_lldpd_port_in_log(logfile)) + .await + .context("time out while discovering lldpd port number")? +} + +async fn find_lldpd_port_in_log(logfile: String) -> Result { + let re = regex::Regex::new(r#""local_addr":"\[::1?\]:([0-9]+)""#).unwrap(); + let mut reader = BufReader::new(File::open(&logfile).await?); + let mut lines = reader.lines(); + loop { + match lines.next_line().await? { + Some(line) => { + if let Some(cap) = re.captures(&line) { + // unwrap on get(1) should be ok, since captures() returns + // `None` if there are no matches found + let port = cap.get(1).unwrap(); + let result = port.as_str().parse::()?; + return Ok(result); + } + } + None => { + sleep(Duration::from_millis(10)).await; + + // We might have gotten a partial line; close the file, reopen + // it, and start reading again from the beginning. + reader = BufReader::new(File::open(&logfile).await?); + lines = reader.lines(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::find_lldpd_port_in_log; + use std::io::Write; + use std::process::Stdio; + use tempfile::NamedTempFile; + + const EXPECTED_PORT: u16 = 12230; + + #[tokio::test] + async fn test_lldpd_in_path() { + // With no arguments, we expect to see the default help message. + tokio::process::Command::new("lldpd") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Cannot find 'lldpd' on PATH. Refer to README.md for installation instructions"); + } + + #[tokio::test] + async fn test_discover_local_listening_port() { + // Write some data to a fake log file + // This line is representative of the kind of output that lldpd currently logs + let line = r#"{"msg":"listening","v":0,"name":"lldpd","level":30,"time":"2025-12-23T00:07:09.226947807Z","hostname":"sled03","pid":10187,"local_addr":"[::]:12230","server_id": +"1","unit":"api-server"}"#; + let mut file = NamedTempFile::new().unwrap(); + writeln!(file, "A garbage line").unwrap(); + writeln!(file, "{}", line).unwrap(); + writeln!(file, "Another garbage line").unwrap(); + file.flush().unwrap(); + + assert_eq!( + find_lldpd_port_in_log(file.path().display().to_string()) + .await + .unwrap(), + EXPECTED_PORT + ); + } +} From e5354b3e7cb16a5efcd999fb7a655c7617d0ba7d Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Tue, 23 Dec 2025 03:42:45 +0000 Subject: [PATCH 7/9] fix missing enum variant --- nexus/reconfigurator/planning/src/example.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index bcf48f2c1ed..96bfb9d8350 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -1424,6 +1424,7 @@ mod tests { | ServiceName::Dendrite | ServiceName::Tfport | ServiceName::BoundaryNtp + | ServiceName::Lldpd | ServiceName::Mgd => { out.insert(service, Err(QueryError::NoRecordsFound)); } From 2af197122bb392ebb5b355481478c3fd7cb72548 Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Tue, 23 Dec 2025 20:01:56 +0000 Subject: [PATCH 8/9] fix tests --- nexus-config/src/nexus_config.rs | 2 ++ nexus/reconfigurator/execution/src/test_utils.rs | 2 ++ nexus/tests/integration_tests/initialization.rs | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/nexus-config/src/nexus_config.rs b/nexus-config/src/nexus_config.rs index 71c8c0350df..3a0be6afdbe 100644 --- a/nexus-config/src/nexus_config.rs +++ b/nexus-config/src/nexus_config.rs @@ -1224,6 +1224,8 @@ mod test { type = "from_dns" [dendrite.switch0] address = "[::1]:12224" + [lldpd.switch0] + address = "[::1]:12230" [mgd.switch0] address = "[::1]:4676" [initial_reconfigurator_config] diff --git a/nexus/reconfigurator/execution/src/test_utils.rs b/nexus/reconfigurator/execution/src/test_utils.rs index 737a2b16b59..d51ade7fc7b 100644 --- a/nexus/reconfigurator/execution/src/test_utils.rs +++ b/nexus/reconfigurator/execution/src/test_utils.rs @@ -118,10 +118,12 @@ pub fn overridables_for_test( .unwrap() .port; let mgd_port = cptestctx.mgd.get(&switch_location).unwrap().port; + let lldpd_port = cptestctx.lldpd.get(&switch_location).unwrap().port; overrides.override_switch_zone_ip(sled_id, ip); overrides.override_dendrite_port(sled_id, dendrite_port); overrides.override_mgs_port(sled_id, mgs_port); overrides.override_mgd_port(sled_id, mgd_port); + overrides.override_lldpd_port(sled_id, lldpd_port); } overrides } diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index b748bbe2985..48bc3584e30 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -165,6 +165,11 @@ async fn test_nexus_boots_before_dendrite() { starter.start_dendrite(SwitchLocation::Switch1).await; info!(log, "Started Dendrite"); + info!(log, "Starting lldp"); + starter.start_lldp(SwitchLocation::Switch0).await; + starter.start_lldp(SwitchLocation::Switch1).await; + info!(log, "Started lldp"); + info!(log, "Starting mgd"); starter.start_mgd(SwitchLocation::Switch0).await; starter.start_mgd(SwitchLocation::Switch1).await; From 3492950d27d3d9660cd19a8767202334b667216c Mon Sep 17 00:00:00 2001 From: Levon Tarver Date: Tue, 23 Dec 2025 20:29:30 +0000 Subject: [PATCH 9/9] cargo fmt --- dev-tools/downloader/src/lib.rs | 8 +++----- nexus/test-utils/src/starter.rs | 16 +++++++--------- test-utils/src/dev/lldp.rs | 13 ++++++++++--- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index bd718dfa810..9a9cc52f0d6 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -884,8 +884,7 @@ impl Downloader<'_> { &checksums_path, ) .await?; - let commit_path = - self.versions_dir.join("lldp_openapi_version"); + let commit_path = self.versions_dir.join("lldp_openapi_version"); let [commit] = get_values_from_file(["COMMIT"], &commit_path).await?; let repo = "oxidecomputer/lldp"; @@ -935,9 +934,8 @@ impl Downloader<'_> { let binaries = [("lldp", &["--no-default-features"][..])]; - let built_binaries = self - .build_from_git("lldp", &commit, &binaries) - .await?; + let built_binaries = + self.build_from_git("lldp", &commit, &binaries).await?; // Copy built binary to binary_dir let dest = binary_dir.join("lldp"); diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 3ca2d3e4e2e..2205a73d26c 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -25,8 +25,8 @@ use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use internal_dns_types::names::ServiceName; use nexus_config::Database; use nexus_config::DpdConfig; -use nexus_config::LldpdConfig; use nexus_config::InternalDns; +use nexus_config::LldpdConfig; use nexus_config::MgdConfig; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_config::NexusConfig; @@ -451,16 +451,14 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { let mgs_addr = SocketAddrV6::new(Ipv6Addr::LOCALHOST, mgs.port, 0, 0).into(); - let dpd_port = self.dendrite.read().unwrap().get(&switch_location).unwrap().port; + let dpd_port = + self.dendrite.read().unwrap().get(&switch_location).unwrap().port; // Set up an instance of lldpd - let lldpd = dev::lldp::LldpdInstance::start( - 0, - dpd_port, - Some(mgs_addr), - ) - .await - .unwrap(); + let lldpd = + dev::lldp::LldpdInstance::start(0, dpd_port, Some(mgs_addr)) + .await + .unwrap(); let port = lldpd.port; self.lldpd.insert(switch_location, lldpd); diff --git a/test-utils/src/dev/lldp.rs b/test-utils/src/dev/lldp.rs index 808927d6b6f..6144c302445 100644 --- a/test-utils/src/dev/lldp.rs +++ b/test-utils/src/dev/lldp.rs @@ -42,7 +42,8 @@ impl LldpdInstance { mgs_address: Option, ) -> Result { let temp_dir = TempDir::new()?; - let listen_addr = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port); + let listen_addr = + SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port); let mut args = vec![ "run".to_string(), @@ -60,8 +61,14 @@ impl LldpdInstance { let child = tokio::process::Command::new("lldpd") .args(&args) .stdin(Stdio::null()) - .stdout(Stdio::from(redirect_file(temp_dir.path(), "lldpd_stdout")?)) - .stderr(Stdio::from(redirect_file(temp_dir.path(), "lldpd_stderr")?)) + .stdout(Stdio::from(redirect_file( + temp_dir.path(), + "lldpd_stdout", + )?)) + .stderr(Stdio::from(redirect_file( + temp_dir.path(), + "lldpd_stderr", + )?)) .spawn() .with_context(|| { format!("failed to spawn `lldpd` (with args: {:?})", &args)