diff --git a/Cargo.lock b/Cargo.lock index 89f89b92..3f7eb82e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ dependencies = [ "log", "mio", "parking_lot", + "rstest", "serde", "serde-aco", "snafu", @@ -221,6 +222,12 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "flexi_logger" version = "0.29.6" @@ -234,6 +241,113 @@ dependencies = [ "thiserror", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + [[package]] name = "heck" version = "0.5.0" @@ -263,6 +377,16 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "io-uring" version = "0.7.2" @@ -376,6 +500,27 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "pin-project-lite" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.92" @@ -405,9 +550,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -417,9 +562,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -428,9 +573,54 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + +[[package]] +name = "rstest" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a2c585be59b6b5dd66a9d2084aa1d8bd52fbdb806eafdeffb52791147862035" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "825ea780781b15345a146be27eaefb05085e337e869bff01b4306a4fd4a9ad5a" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] [[package]] name = "scopeguard" @@ -438,6 +628,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.215" @@ -486,6 +682,15 @@ dependencies = [ "syn", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -550,11 +755,28 @@ dependencies = [ "syn", ] +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "toml_datetime", + "winnow", +] + [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "utf8parse" @@ -770,6 +992,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "zerocopy" version = "0.8.13" diff --git a/Cargo.toml b/Cargo.toml index 08d1008d..b04f6ea9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ assert_matches = "1" proc-macro2 = "1" syn = { version = "2", features = ["full"] } quote = { version = "1" } +rstest = "0.23" [profile.release] lto = true diff --git a/alioth/Cargo.toml b/alioth/Cargo.toml index 68fecab1..2629e6f3 100644 --- a/alioth/Cargo.toml +++ b/alioth/Cargo.toml @@ -28,3 +28,4 @@ io-uring = "0.7" [dev-dependencies] assert_matches.workspace = true +rstest.workspace = true diff --git a/alioth/src/hv/kvm/vmexit.rs b/alioth/src/hv/kvm/vmexit.rs index 537144a6..2ef5f63f 100644 --- a/alioth/src/hv/kvm/vmexit.rs +++ b/alioth/src/hv/kvm/vmexit.rs @@ -23,11 +23,10 @@ impl KvmVcpu { #[cfg(target_endian = "little")] pub(super) fn handle_mmio(&mut self) -> Result { let kvm_mmio = unsafe { &self.kvm_run.exit.mmio }; - let data = u64::from_ne_bytes(kvm_mmio.data) & u64::MAX >> (64 - (kvm_mmio.len << 3)); let exit = VmExit::Mmio { addr: kvm_mmio.phys_addr, write: if kvm_mmio.is_write > 0 { - Some(data) + Some(u64::from_ne_bytes(kvm_mmio.data)) } else { None }, diff --git a/alioth/src/mem/emulated.rs b/alioth/src/mem/emulated.rs index a9f19052..bc9a2009 100644 --- a/alioth/src/mem/emulated.rs +++ b/alioth/src/mem/emulated.rs @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cmp::min; use std::fmt::Debug; use std::sync::Arc; use crate::mem::addressable::{Addressable, SlotBackend}; use crate::mem::{Memory, Result}; -use parking_lot::RwLock; +use crate::utils::truncate_u64; pub trait ChangeLayout: Debug + Send + Sync + 'static { fn change(&self, memory: &Memory) -> Result<()>; @@ -118,7 +119,7 @@ pub struct MmioBus where R: Debug + SlotBackend, { - pub(crate) inner: RwLock>, + pub(crate) inner: Addressable, } impl Default for MmioBus @@ -136,38 +137,201 @@ where { pub fn new() -> MmioBus { Self { - inner: RwLock::new(Addressable::new()), + inner: Addressable::new(), } } pub fn is_empty(&self) -> bool { - self.inner.read().is_empty() + self.inner.is_empty() } - pub fn add(&self, addr: u64, range: R) -> Result<()> { - let mut inner = self.inner.write(); - inner.add(addr, range)?; + pub fn add(&mut self, addr: u64, range: R) -> Result<()> { + self.inner.add(addr, range)?; Ok(()) } - pub(super) fn remove(&self, addr: u64) -> Result { - let mut inner = self.inner.write(); - inner.remove(addr) + pub(super) fn remove(&mut self, addr: u64) -> Result { + self.inner.remove(addr) } pub fn read(&self, addr: u64, size: u8) -> Result { - let inner = self.inner.read(); - match inner.search(addr) { - Some((start, dev)) => dev.read(addr - start, size), - None => Ok(0), + let mut count = 0; + let mut val = 0; + let size = size as u64; + while count < size { + let base_addr = addr + count; + let Some((start, dev)) = self.inner.search_next(base_addr) else { + break; + }; + count += start.saturating_sub(base_addr); + let offset = base_addr.saturating_sub(start); + let mut read_size = min(Mmio::size(dev) - offset, size.saturating_sub(count)); + if read_size == 0 { + break; + } + read_size = min(read_size, 1 << read_size.trailing_zeros()); + if offset > 0 { + read_size = min(read_size, 1 << offset.trailing_zeros()); + } + val |= truncate_u64(dev.read(offset, read_size as u8)?, read_size) << (count << 3); + count += read_size; } + Ok(val) } pub fn write(&self, addr: u64, size: u8, val: u64) -> Result { - let inner = self.inner.read(); - match inner.search(addr) { - Some((start, dev)) => dev.write(addr - start, size, val), - None => Ok(Action::None), + let mut count = 0; + let size = size as u64; + let mut action = Action::None; + while count < size { + let base_addr = addr + count; + let Some((start, dev)) = self.inner.search_next(base_addr) else { + break; + }; + count += start.saturating_sub(base_addr); + let offset = base_addr.saturating_sub(start); + let mut write_size = min(Mmio::size(dev) - offset, size.saturating_sub(count)); + if write_size == 0 { + break; + } + write_size = min(write_size, 1 << write_size.trailing_zeros()); + if offset > 0 { + write_size = min(write_size, 1 << offset.trailing_zeros()); + } + let write_val = truncate_u64(val >> (count << 3), write_size); + let r = dev.write(offset, write_size as u8, write_val)?; + if matches!(action, Action::None) { + action = r + } else { + // TODO: handle multiple side effects caused by a single write + log::error!( + "Write {write_val:#x} to {:#x}: dropped: {action:#x?}", + start + offset + ); + } + count += write_size; + } + Ok(action) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use parking_lot::Mutex; + use rstest::{fixture, rstest}; + + use crate::mem::emulated::{Action, Mmio, MmioBus}; + use crate::mem::Result; + + #[derive(Debug)] + struct TestRange { + size: u64, + val: Mutex, + } + + impl Mmio for TestRange { + fn size(&self) -> u64 { + self.size + } + fn read(&self, offset: u64, _size: u8) -> Result { + let val = *self.val.lock() >> (offset << 3); + Ok(val) + } + fn write(&self, offset: u64, size: u8, val: u64) -> Result { + assert_eq!(size.count_ones(), 1); + assert!(offset.trailing_zeros() >= size.trailing_zeros()); + let v = &mut *self.val.lock(); + let shift = offset << 3; + *v &= !(((1 << (size << 3)) - 1) << shift); + *v |= val << shift; + Ok(Action::None) + } + } + + // Creates a bus containing the following values: + // | 0x01 | 0x23 | 0x67 0x45 | 0xef 0xcd 0xab 0x89 + // | 0x34 0x12 0xcd 0xab + #[fixture] + fn fixture_mmio_bus() -> MmioBus { + let mut bus: MmioBus = MmioBus::new(); + for (offset, size, val) in [ + (0x0, 1, 0xffff_ffff_ffff_ff01), + (0x1, 1, 0xffff_ffff_ffff_ff23), + (0x2, 2, 0xffff_ffff_ffff_4567), + (0x4, 4, 0xffff_ffff_89ab_cdef), + (0xc, 4, 0xffff_ffff_abcd_1234), + ] { + bus.add( + offset, + Arc::new(TestRange { + size, + val: Mutex::new(val), + }), + ) + .unwrap(); } + bus + } + + #[rstest] + #[case(0x0, 1, 0x01)] + #[case(0x0, 2, 0x2301)] + #[case(0x0, 3, 0x672301)] + #[case(0x0, 4, 0x45672301)] + #[case(0x0, 8, 0x89ab_cdef_4567_2301)] + #[case(0x1, 1, 0x23)] + #[case(0x1, 2, 0x6723)] + #[case(0x1, 3, 0x45_6723)] + #[case(0x1, 4, 0xef45_6723)] + #[case(0x1, 8, 0x89_abcd_ef45_6723)] + #[case(0x4, 8, 0x89ab_cdef)] + #[case(0x6, 1, 0xab)] + #[case(0x6, 2, 0x89ab)] + #[case(0x6, 4, 0x89ab)] + #[case(0x8, 1, 0x0)] + #[case(0x8, 4, 0x0)] + #[case(0x8, 5, 0x34_0000_0000)] + #[case(0x8, 8, 0xabcd_1234_0000_0000)] + #[case(0xa, 8, 0x0000_abcd_1234_0000)] + fn test_mmio_bus_read( + fixture_mmio_bus: MmioBus, + #[case] addr: u64, + #[case] size: u8, + #[case] val: u64, + ) { + assert_eq!(fixture_mmio_bus.read(addr, size).unwrap(), val) + } + + #[rstest] + #[case(0x0, 1, 0x3210, 0x89ab_cdef_4567_2310)] + #[case(0x0, 2, 0x3210, 0x89ab_cdef_4567_3210)] + #[case(0x0, 3, 0x763201, 0x89ab_cdef_4576_3201)] + #[case(0x0, 4, 0x10, 0x89ab_cdef_0000_0010)] + #[case(0x0, 8, 0x10, 0x10)] + #[case(0x1, 1, 0x32, 0x89_abcd_ef45_6732)] + #[case(0x1, 2, 0x7632, 0x89_abcd_ef45_7632)] + #[case(0x1, 3, 0x1254_7632, 0x89_abcd_ef54_7632)] + #[case(0x1, 4, 0xfe54_7632, 0x89_abcd_fe54_7632)] + #[case(0x1, 8, 0x0, 0x0)] + #[case(0x4, 8, 0x1234_89ab_cdef, 0x89ab_cdef)] + #[case(0x6, 1, 0xba, 0x1234_0000_0000_89ba)] + #[case(0x6, 2, 0x98ba, 0x1234_0000_0000_98ba)] + #[case(0x6, 4, 0xff_ab98, 0x1234_0000_0000_ab98)] + #[case(0x8, 1, 0xff, 0xabcd_1234_0000_0000)] + #[case(0x8, 4, 0xffff, 0xabcd_1234_0000_0000)] + #[case(0x8, 5, 0xfe_1234_0000, 0xabcd_12fe_0000_0000)] + #[case(0x8, 8, 0x5678_cdfe_1234_0000, 0x5678_cdfe_0000_0000)] + #[case(0xa, 8, 0xcdfe_1234_abcd, 0xcdfe_1234_0000)] + fn test_mmio_bus_write( + fixture_mmio_bus: MmioBus, + #[case] addr: u64, + #[case] size: u8, + #[case] val: u64, + #[case] expected: u64, + ) { + assert!(fixture_mmio_bus.write(addr, size, val).is_ok()); + assert_eq!(fixture_mmio_bus.read(addr, 8).unwrap(), expected); } } diff --git a/alioth/src/mem/mem.rs b/alioth/src/mem/mem.rs index b0572bdb..98df54b5 100644 --- a/alioth/src/mem/mem.rs +++ b/alioth/src/mem/mem.rs @@ -16,7 +16,7 @@ use std::any::type_name; use std::fmt::Debug; use std::sync::Arc; -use parking_lot::Mutex; +use parking_lot::{Mutex, RwLock}; use serde::Deserialize; use serde_aco::Help; use snafu::Snafu; @@ -255,27 +255,29 @@ struct LayoutCallbacks { updated: Vec>, } +// lock order: region -> callbacks -> bus #[derive(Debug)] pub struct Memory { - ram_bus: Arc, - mmio_bus: MmioBus, regions: Mutex>>, + callbacks: Mutex, + ram_bus: Arc, + mmio_bus: RwLock, vm_memory: Box, - io_bus: MmioBus, + + io_bus: RwLock, io_regions: Mutex>>, - callbacks: Mutex, } impl Memory { pub fn new(vm_memory: impl VmMemory) -> Self { Memory { - ram_bus: Arc::new(RamBus::new()), - mmio_bus: MmioBus::new(), regions: Mutex::new(Addressable::new()), + callbacks: Mutex::new(LayoutCallbacks::default()), + ram_bus: Arc::new(RamBus::new()), + mmio_bus: RwLock::new(MmioBus::new()), vm_memory: Box::new(vm_memory), - io_bus: MmioBus::new(), + io_bus: RwLock::new(MmioBus::new()), io_regions: Mutex::new(Addressable::new()), - callbacks: Mutex::new(LayoutCallbacks::default()), } } @@ -299,9 +301,9 @@ impl Memory { pub fn register_update_callback(&self, callback: Box) -> Result<()> { let _regions = self.regions.lock(); + let mut callbacks = self.callbacks.lock(); let ram = self.ram_bus.lock_layout(); callback.ram_updated(&ram)?; - let mut callbacks = self.callbacks.lock(); callbacks.updated.push(callback); Ok(()) } @@ -343,7 +345,10 @@ impl Memory { for range in ®ion.ranges { let gpa = addr + offset; match range { - MemRange::Emulated(r) => self.mmio_bus.add(gpa, r.clone())?, + MemRange::Emulated(r) => { + let mut mmio_bus = self.mmio_bus.write(); + mmio_bus.add(gpa, r.clone())? + } MemRange::Ram(r) => { self.map_to_vm(gpa, r)?; for callback in &callbacks.changed { @@ -378,7 +383,8 @@ impl Memory { let gpa = addr + offset; match range { MemRange::Emulated(_) => { - self.mmio_bus.remove(gpa)?; + let mut mmio_bus = self.mmio_bus.write(); + mmio_bus.remove(gpa)?; } MemRange::Ram(r) => { self.ram_bus.remove(gpa)?; @@ -451,7 +457,8 @@ impl Memory { pub fn add_io_region(&self, port: u16, region: Arc) -> Result<()> { let mut regions = self.io_regions.lock(); regions.add(port as u64, region.clone())?; - self.io_bus.add(port as u64, region.range.clone())?; + let mut io_bus = self.io_bus.write(); + io_bus.add(port as u64, region.range.clone())?; let callbacks = region.callbacks.lock(); for callback in callbacks.iter() { callback.mapped(port as u64)?; @@ -460,7 +467,8 @@ impl Memory { } fn unmap_io_region(&self, port: u16, region: &IoRegion) -> Result<()> { - self.io_bus.remove(port as u64)?; + let mut io_bus = self.io_bus.write(); + io_bus.remove(port as u64)?; let callbacks = region.callbacks.lock(); for callback in callbacks.iter() { callback.unmapped()?; @@ -542,21 +550,25 @@ impl Memory { } pub fn handle_mmio(&self, gpa: u64, write: Option, size: u8) -> Result { + let mmio_bus = self.mmio_bus.read(); if let Some(val) = write { - let action = self.mmio_bus.write(gpa, size, val)?; + let action = mmio_bus.write(gpa, size, val)?; + drop(mmio_bus); self.handle_action(action) } else { - let data = self.mmio_bus.read(gpa, size)?; + let data = mmio_bus.read(gpa, size)?; Ok(VmEntry::Mmio { data }) } } pub fn handle_io(&self, port: u16, write: Option, size: u8) -> Result { + let io_bus = self.io_bus.read(); if let Some(val) = write { - let action = self.io_bus.write(port as u64, size, val as u64)?; + let action = io_bus.write(port as u64, size, val as u64)?; + drop(io_bus); self.handle_action(action) } else { - let data = self.io_bus.read(port as u64, size)? as u32; + let data = io_bus.read(port as u64, size)? as u32; Ok(VmEntry::Io { data }) } } diff --git a/alioth/src/pci/cap.rs b/alioth/src/pci/cap.rs index cafbccbc..c067b4bd 100644 --- a/alioth/src/pci/cap.rs +++ b/alioth/src/pci/cap.rs @@ -23,8 +23,9 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use crate::hv::IrqFd; use crate::mem::addressable::SlotBackend; use crate::mem::emulated::{Action, Mmio, MmioBus}; -use crate::pci::config::DeviceHeader; +use crate::pci::config::{DeviceHeader, PciConfigArea}; use crate::pci::Error; +use crate::utils::truncate_u64; use crate::{align_up, impl_mmio_for_zerocopy, mem}; #[repr(u8)] @@ -52,6 +53,32 @@ bitfield! { pub id, _: 15,0; } +#[repr(C)] +#[derive(Debug, Default, Clone)] +pub struct NullCap { + pub next: u8, + pub size: u8, +} + +impl Mmio for NullCap { + fn read(&self, offset: u64, size: u8) -> mem::Result { + let shift = std::cmp::min(63, offset << 3); + let val = ((self.next as u64) << 8) >> shift; + Ok(truncate_u64(val, size as u64)) + } + + fn write(&self, _offset: u64, _size: u8, _val: u64) -> mem::Result { + Ok(Action::None) + } + fn size(&self) -> u64 { + self.size as u64 + } +} + +impl PciConfigArea for NullCap { + fn reset(&self) {} +} + bitfield! { #[derive(Copy, Clone, Default, FromBytes, Immutable, IntoBytes, KnownLayout)] #[repr(C)] @@ -59,20 +86,20 @@ bitfield! { impl Debug; pub enable, set_enable: 0; pub multi_msg_cap, set_multi_msg_cap: 3, 1; - pub multi_msg_enable, set_multi_msg_enable: 6, 4; - pub addr_64, set_addr_64: 7; - pub per_vector_masking, set_per_vector_masking: 8; + pub multi_msg, set_multi_msg: 6, 4; + pub addr_64_cap, set_addr_64_cap: 7; + pub per_vector_masking_cap, set_per_vector_masking_cap: 8; pub ext_msg_data_cap, set_ext_msg_data_cap: 9; - pub ext_msg_data_enable, set_ext_msg_data_enable: 10; + pub ext_msg_data, set_ext_msg_data: 10; } impl MsiMsgCtrl { - pub fn cap_size(&self) -> usize { + pub fn cap_size(&self) -> u8 { let mut size = 12; - if self.addr_64() { + if self.addr_64_cap() { size += 4; } - if self.per_vector_masking() { + if self.per_vector_masking_cap() { size += 8; } size @@ -85,6 +112,7 @@ pub struct MsiCapHdr { pub header: PciCapHdr, pub control: MsiMsgCtrl, } +impl_mmio_for_zerocopy!(MsiCapHdr); bitfield! { #[derive(Copy, Clone, Default, FromBytes, Immutable, IntoBytes, KnownLayout)] @@ -158,9 +186,8 @@ impl Default for MsixTableEntry { } } -pub trait PciCap: Mmio { +pub trait PciCap: PciConfigArea { fn set_next(&mut self, val: u8); - fn reset(&self); } impl SlotBackend for Box { @@ -206,8 +233,7 @@ impl PciCapList { } pub fn reset(&self) { - let inner = self.inner.inner.read(); - for (_, cap) in inner.iter() { + for (_, cap) in self.inner.inner.iter() { cap.reset(); } } @@ -230,7 +256,7 @@ impl Mmio for PciCapList { impl TryFrom>> for PciCapList { type Error = Error; fn try_from(caps: Vec>) -> Result { - let bus = MmioBus::new(); + let mut bus = MmioBus::new(); let mut ptr = size_of::() as u64; let num_caps = caps.len(); for (index, mut cap) in caps.into_iter().enumerate() { @@ -273,11 +299,7 @@ impl Mmio for MsixCapMmio { } } -impl PciCap for MsixCapMmio { - fn set_next(&mut self, val: u8) { - self.cap.write().header.next = val; - } - +impl PciConfigArea for MsixCapMmio { fn reset(&self) { let mut cap = self.cap.write(); cap.control.set_enabled(false); @@ -285,6 +307,12 @@ impl PciCap for MsixCapMmio { } } +impl PciCap for MsixCapMmio { + fn set_next(&mut self, val: u8) { + self.cap.write().header.next = val; + } +} + #[derive(Debug)] pub enum MsixTableMmioEntry { Entry(MsixTableEntry), @@ -420,3 +448,30 @@ where Ok(Action::None) } } + +#[cfg(test)] +mod tests { + use assert_matches::assert_matches; + use rstest::rstest; + + use crate::mem::emulated::Mmio; + use crate::pci::cap::NullCap; + + #[rstest] + #[case(0x0, 1, 0x0)] + #[case(0x0, 2, 0x60_00)] + #[case(0x0, 4, 0x60_00)] + #[case(0x1, 1, 0x60)] + #[case(0x1, 2, 0x60)] + #[case(0x1, 2, 0x60)] + #[case(0x2, 1, 0x0)] + #[case(0x2, 2, 0x0)] + #[case(0xb, 1, 0x0)] + fn test_null_cap(#[case] offset: u64, #[case] size: u8, #[case] val: u64) { + let null_cap = NullCap { + next: 0x60, + size: 0xc, + }; + assert_matches!(null_cap.read(offset, size), Ok(v) if v == val); + } +} diff --git a/alioth/src/pci/config.rs b/alioth/src/pci/config.rs index 68dc2450..80f329c5 100644 --- a/alioth/src/pci/config.rs +++ b/alioth/src/pci/config.rs @@ -21,11 +21,36 @@ use macros::Layout; use parking_lot::RwLock; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; +use crate::mem::addressable::SlotBackend; use crate::mem::emulated::{Action, ChangeLayout, Mmio}; use crate::pci::cap::PciCapList; use crate::pci::{Bdf, PciBar}; use crate::{assign_bits, impl_mmio_for_zerocopy, mask_bits, mem}; +pub trait PciConfigArea: Mmio { + fn reset(&self); +} + +impl Mmio for Box { + fn read(&self, offset: u64, size: u8) -> mem::Result { + Mmio::read(self.as_ref(), offset, size) + } + + fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result { + Mmio::write(self.as_ref(), offset, size, val) + } + + fn size(&self) -> u64 { + Mmio::size(self.as_ref()) + } +} + +impl SlotBackend for Box { + fn size(&self) -> u64 { + Mmio::size(self.as_ref()) + } +} + #[derive(Clone, Copy, Default, IntoBytes, FromBytes, KnownLayout, Immutable)] #[repr(transparent)] pub struct Command(u16); @@ -380,11 +405,6 @@ impl EmulatedHeader { let mut header = self.data.write(); header.set_command(command) } - - pub fn reset(&self) { - let mut header = self.data.write(); - header.set_command(Command::empty()); - } } impl Mmio for EmulatedHeader { @@ -409,6 +429,13 @@ impl Mmio for EmulatedHeader { } } +impl PciConfigArea for EmulatedHeader { + fn reset(&self) { + let mut header = self.data.write(); + header.set_command(Command::empty()); + } +} + pub trait PciConfig: Mmio { fn get_header(&self) -> &EmulatedHeader; fn reset(&self); diff --git a/alioth/src/utils/utils.rs b/alioth/src/utils/utils.rs index 31d869b4..a60fc6e1 100644 --- a/alioth/src/utils/utils.rs +++ b/alioth/src/utils/utils.rs @@ -18,6 +18,10 @@ pub mod endian; #[cfg(target_os = "linux")] pub mod ioctls; +pub fn truncate_u64(val: u64, size: u64) -> u64 { + val & u64::MAX >> (64 - (size << 3)) +} + #[macro_export] macro_rules! align_up { ($num:expr, $bits:expr) => {{ diff --git a/alioth/src/vfio/device.rs b/alioth/src/vfio/device.rs index baf75573..50b155f2 100644 --- a/alioth/src/vfio/device.rs +++ b/alioth/src/vfio/device.rs @@ -19,7 +19,10 @@ use std::os::fd::AsRawFd; use std::os::unix::fs::FileExt; use crate::mem; -use crate::vfio::bindings::{VfioDeviceInfo, VfioIrqInfo, VfioIrqSet, VfioRegionInfo}; +use crate::vfio::bindings::{ + VfioDeviceInfo, VfioIrqInfo, VfioIrqSet, VfioIrqSetData, VfioIrqSetFlag, VfioPciIrq, + VfioRegionInfo, +}; use crate::vfio::ioctls::{ vfio_device_get_info, vfio_device_get_irq_info, vfio_device_get_region_info, vfio_device_reset, vfio_device_set_irqs, @@ -63,6 +66,18 @@ pub trait Device: Debug + Send + Sync + 'static { Ok(()) } + fn disable_all_irqs(&self, index: VfioPciIrq) -> Result<()> { + let vfio_irq_disable_all = VfioIrqSet { + argsz: size_of::>() as u32, + flags: VfioIrqSetFlag::DATA_NONE | VfioIrqSetFlag::ACTION_TRIGGER, + index: index.raw(), + start: 0, + count: 0, + data: VfioIrqSetData { eventfds: [] }, + }; + self.set_irqs(&vfio_irq_disable_all) + } + fn reset(&self) -> Result<()> { unsafe { vfio_device_reset(self.fd()) }?; Ok(()) diff --git a/alioth/src/vfio/pci.rs b/alioth/src/vfio/pci.rs index cd4fbfa1..8d8adc01 100644 --- a/alioth/src/vfio/pci.rs +++ b/alioth/src/vfio/pci.rs @@ -12,31 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cmp::min; use std::fs::File; +use std::iter::zip; use std::mem::size_of; use std::ops::Range; use std::os::fd::{AsFd, AsRawFd}; use std::os::unix::fs::FileExt; -use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::atomic::AtomicU64; use std::sync::Arc; use libc::{PROT_READ, PROT_WRITE}; +use macros::Layout; use parking_lot::{Mutex, RwLock}; use snafu::ResultExt; -use zerocopy::{transmute, FromBytes}; +use zerocopy::{transmute, FromBytes, Immutable, IntoBytes}; use crate::errors::boxed_debug_trace; use crate::hv::{IrqFd, MsiSender}; -use crate::mem::addressable::{Addressable, SlotBackend}; -use crate::mem::emulated::{Action, Mmio}; +use crate::mem::emulated::{Action, Mmio, MmioBus}; use crate::mem::mapped::ArcMemPages; use crate::mem::{IoRegion, MemRange, MemRegion, MemRegionEntry, MemRegionType}; use crate::pci::cap::{ - MsiCapHdr, MsixCap, MsixTableEntry, MsixTableMmio, MsixTableMmioEntry, PciCapHdr, PciCapId, + MsiCapHdr, MsiMsgCtrl, MsixCap, MsixCapMmio, MsixTableEntry, MsixTableMmio, MsixTableMmioEntry, + NullCap, PciCapHdr, PciCapId, }; use crate::pci::config::{ Command, CommonHeader, ConfigHeader, DeviceHeader, EmulatedHeader, HeaderData, HeaderType, - PciConfig, Status, BAR_IO, BAR_MEM64, + PciConfig, PciConfigArea, Status, BAR_IO, BAR_MEM64, }; use crate::pci::{self, Bdf, Pci, PciBar}; use crate::vfio::bindings::{ @@ -45,7 +48,7 @@ use crate::vfio::bindings::{ }; use crate::vfio::device::Device; use crate::vfio::{error, Result}; -use crate::{align_down, align_up, assign_bits, mask_bits, mem}; +use crate::{align_down, align_up, impl_mmio_for_zerocopy, mem}; fn round_up_range(range: Range) -> Range { (align_down!(range.start, 12))..(align_up!(range.end, 12)) @@ -199,17 +202,6 @@ where } } -#[derive(Debug)] -struct MaskedCap { - size: usize, -} - -impl SlotBackend for MaskedCap { - fn size(&self) -> u64 { - self.size as u64 - } -} - #[derive(Debug)] struct VfioDev { name: Arc, @@ -217,96 +209,40 @@ struct VfioDev { } #[derive(Debug)] -pub struct ExtraConfig { - msix_msg_ctrl_hi: AtomicU8, - msix_msg_ctrl_offset: usize, - masked_caps: Addressable, - +struct PthConfigArea { + offset: u64, // offset to dev + size: u64, dev: Arc>, - offset: u64, - size: usize, } -impl ExtraConfig +impl Mmio for PthConfigArea where D: Device, { - fn write(&self, offset: usize, size: u8, val: u64) -> mem::Result<()> { - let name = &self.dev.name; - let mut masks = [0; 8]; - let vals = val.to_ne_bytes(); - for index in 0..(size as usize) { - let pos = offset + index; - if pos == self.msix_msg_ctrl_offset + 1 { - let mut msix_msg_ctrl_hi = self.msix_msg_ctrl_hi.load(Ordering::Acquire); - const MASKED: u8 = 1 << 6; - const ENABLED: u8 = 1 << 7; - assign_bits!(msix_msg_ctrl_hi, vals[index], MASKED | ENABLED); - // TODO trigger mask / enable interrupt - self.msix_msg_ctrl_hi - .store(msix_msg_ctrl_hi, Ordering::Release); - log::trace!("{name}: msix_msg_ctrl_hi -> {msix_msg_ctrl_hi:#x?}",); - masks[index] = 0xff; - } else if pos == self.msix_msg_ctrl_offset - || self.masked_caps.search(pos as u64).is_some() - { - masks[index] = 0xff; - } - } - let mask = u64::from_ne_bytes(masks); - log::trace!( - "{name}: write config: val=0x{val:0width$x}, offset={offset:#05x}, size={size}, mask=0x{mask:0width$x}", - width = 2 * size as usize - ); - if mask.trailing_ones() == (size << 3) as u32 { - return Ok(()); - } - let cdev = &self.dev.dev; - let masked_val = if mask == 0 { - val - } else { - let real_val = cdev.read(self.offset + offset as u64, size)?; - mask_bits!(val, real_val, mask) - }; - cdev.write(self.offset + offset as u64, size, masked_val) - } - - fn read(&self, offset: usize, size: u8) -> mem::Result { - let mut emulated_bytes = [0; 8]; - let mut masks = [0; 8]; - for index in 0..(size as usize) { - let pos = offset + index; - if pos == self.msix_msg_ctrl_offset + 1 { - emulated_bytes[index] = self.msix_msg_ctrl_hi.load(Ordering::Acquire); - masks[index] = 0xff; - } else if let Some((start, _)) = self.masked_caps.search(pos as u64) { - if pos != start as usize + PciCapHdr::OFFSET_NEXT { - emulated_bytes[index] = 0; - masks[index] = 0xff; - } - } - } - let mask = u64::from_ne_bytes(masks); - let emulated_val = u64::from_ne_bytes(emulated_bytes); - let ret = if mask.trailing_ones() == (size << 3) as u32 { - emulated_val - } else { - let real_val = self.dev.dev.read(self.offset + offset as u64, size)?; - mask_bits!(real_val, emulated_val, mask) - }; - log::trace!( - "{}: read config: offset={offset:#05x}, size={size}, mask=0x{mask:0width$x}, emulated_val=0x{emulated_val:0width$x}, ret=0x{ret:0width$x}", - self.dev.name, - width = 2 * size as usize - ); - Ok(ret) + fn size(&self) -> u64 { + self.size + } + fn read(&self, offset: u64, size: u8) -> mem::Result { + self.dev.dev.read(self.offset + offset, size) + } + fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result { + self.dev.dev.write(self.offset + offset, size, val)?; + Ok(Action::None) } } +impl PciConfigArea for PthConfigArea +where + D: Device, +{ + fn reset(&self) {} +} + #[derive(Debug)] pub struct PciPthConfig { header: EmulatedHeader, - extra: ExtraConfig, + extra: MmioBus>, + dev: Arc>, } impl Mmio for PciPthConfig @@ -317,20 +253,19 @@ where if offset < self.header.size() { Mmio::read(&self.header, offset, size) } else { - self.extra.read(offset as usize, size) + self.extra.read(offset, size) } } fn size(&self) -> u64 { - self.extra.size as u64 + 4096 } fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result { if offset < self.header.size() { Mmio::write(&self.header, offset, size, val) } else { - self.extra.write(offset as usize, size, val)?; - Ok(Action::None) + self.extra.write(offset, size, val) } } } @@ -345,7 +280,9 @@ where fn reset(&self) { self.header.reset(); - self.extra.msix_msg_ctrl_hi.store(0, Ordering::Release) + for (_, area) in self.extra.inner.iter() { + area.reset(); + } } } @@ -415,9 +352,11 @@ where pub fn new(name: Arc, dev: D, msi_sender: M) -> Result> { let cdev = Arc::new(VfioDev { dev, name }); + let msi_sender = Arc::new(msi_sender); + let region_config = cdev.dev.get_region_info(VfioPciRegion::CONFIG.raw())?; - let pci_command = Command::MEM | Command::BUS_MASTER | Command::INTX_DISABLE; + let pci_command = Command::IO | Command::MEM | Command::BUS_MASTER | Command::INTX_DISABLE; cdev.dev.write( region_config.offset + CommonHeader::OFFSET_COMMAND as u64, CommonHeader::SIZE_COMMAND as u8, @@ -437,9 +376,9 @@ where dev_header.intx_pin = 0; dev_header.common.command = Command::empty(); - let mut msix_cap = None; - let mut msix_msg_ctrl_offset = 0; - let mut masked_caps = Addressable::new(); + let mut masked_caps: Vec<(u64, Box)> = vec![]; + let mut msix_info = None; + let mut msi_info = None; if dev_header.common.status.contains(Status::CAP) { let mut cap_offset = dev_header.capability_pointer as usize; @@ -450,29 +389,105 @@ where }; let (cap_header, _) = PciCapHdr::ref_from_prefix(cap_buf).unwrap(); if cap_header.id == PciCapId::Msix as u8 { - if let Ok((c, _)) = MsixCap::read_from_prefix(cap_buf) { - msix_cap = Some(c) - } - msix_msg_ctrl_offset = cap_offset + MsixCap::OFFSET_CONTROL; + let Ok((mut c, _)) = MsixCap::read_from_prefix(cap_buf) else { + log::error!( + "{}: MSIX capability is at an invalid offset: {cap_offset:#x}", + cdev.name + ); + continue; + }; + c.control.set_enabled(false); + c.control.set_masked(false); + msix_info = Some((cap_offset, c.clone())); } else if cap_header.id == PciCapId::Msi as u8 { - let Ok((c, _)) = MsiCapHdr::read_from_prefix(cap_buf) else { + let Ok((mut c, _)) = MsiCapHdr::read_from_prefix(cap_buf) else { log::error!( "{}: MSI capability is at an invalid offset: {cap_offset:#x}", cdev.name ); continue; }; - log::trace!("{}: hiding MSI cap at {cap_offset:#x}", cdev.name); - masked_caps.add( - cap_offset as u64, - MaskedCap { - size: c.control.cap_size(), - }, - )?; + log::info!("{}: MSI cap header: {c:#x?}", cdev.name); + c.control.set_enable(false); + c.control.set_ext_msg_data_cap(true); + let multi_msg_cap = min(5, c.control.multi_msg_cap()); + c.control.set_multi_msg_cap(multi_msg_cap); + msi_info = Some((cap_offset, c)); } cap_offset = cap_header.next as usize; } } + + let mut msix_cap = None; + if let Some((offset, cap)) = msix_info { + msix_cap = Some(cap.clone()); + let msix_cap_mmio = MsixCapMmio { + cap: RwLock::new(cap), + }; + masked_caps.push((offset as u64, Box::new(msix_cap_mmio))); + if let Some((offset, hdr)) = msi_info { + let null_cap = NullCap { + size: hdr.control.cap_size(), + next: hdr.header.next, + }; + masked_caps.push((offset as u64, Box::new(null_cap))); + } + } else if let Some((offset, hdr)) = msi_info { + let count = 1 << hdr.control.multi_msg_cap(); + let irqfds = (0..count) + .map(|_| msi_sender.create_irqfd()) + .collect::, _>>()?; + + let mut eventfds = [-1; 32]; + for (fd, irqfd) in zip(&mut eventfds, &irqfds) { + *fd = irqfd.as_fd().as_raw_fd(); + } + let set_eventfd = VfioIrqSet { + argsz: (size_of::>() + size_of::() * count) as u32, + flags: VfioIrqSetFlag::DATA_EVENTFD | VfioIrqSetFlag::ACTION_TRIGGER, + index: VfioPciIrq::MSI.raw(), + start: 0, + count: count as u32, + data: VfioIrqSetData { eventfds }, + }; + cdev.dev.set_irqs(&set_eventfd)?; + + let msi_cap_mmio = MsiCapMmio:: { + cap: RwLock::new((hdr, MsiCapBody { data: [0; 4] })), + dev: cdev.clone(), + irqfds, + }; + masked_caps.push((offset as u64, Box::new(msi_cap_mmio))); + } + + let mut extra_areas: MmioBus> = MmioBus::new(); + masked_caps.sort_by_key(|(offset, _)| *offset); + let mut area_end = 0x40; + for (offset, cap) in masked_caps { + if area_end < offset { + extra_areas.add( + area_end, + Box::new(PthConfigArea { + offset: region_config.offset + area_end, + size: offset - area_end, + dev: cdev.clone(), + }), + )?; + } + area_end = offset + Mmio::size(&*cap); + extra_areas.add(offset, cap)?; + } + if area_end < region_config.size { + extra_areas.add( + area_end, + Box::new(PthConfigArea { + offset: region_config.offset + area_end, + size: region_config.size - area_end, + dev: cdev.clone(), + }), + )?; + } + let config_header = ConfigHeader::Device(dev_header); cdev.dev.reset()?; @@ -486,7 +501,6 @@ where let msix_table = Arc::new(MsixTableMmio { entries: msix_entries, }); - let msi_sender = Arc::new(msi_sender); let mut bars = [const { PciBar::Empty }; 6]; let mut bar_masks = [0u32; 6]; @@ -547,32 +561,24 @@ where })), bars, }, - extra: ExtraConfig { - msix_msg_ctrl_hi: AtomicU8::new(0), - msix_msg_ctrl_offset, - masked_caps, - dev: cdev, - offset: region_config.offset, - size: region_config.size as usize, - }, + extra: extra_areas, + dev: cdev, }, msix_table, }) } fn reset(&self) -> Result<()> { - let disable_msix = VfioIrqSet { - argsz: size_of::>() as u32, - flags: VfioIrqSetFlag::DATA_NONE | VfioIrqSetFlag::ACTION_TRIGGER, - index: VfioPciIrq::MSIX.raw(), - start: 0, - count: 0, - data: VfioIrqSetData { eventfds: [] }, - }; - self.config.extra.dev.dev.set_irqs(&disable_msix)?; + let is_irqfd = |e| matches!(e, &MsixTableMmioEntry::IrqFd(_)); + if self.msix_table.entries.read().iter().any(is_irqfd) { + let dev = &self.config.dev; + if let Err(e) = dev.dev.disable_all_irqs(VfioPciIrq::MSIX) { + log::error!("{}: failed to disable MSIX IRQs: {e:?}", dev.name) + } + } self.msix_table.reset(); - self.config.extra.dev.dev.reset() + self.config.dev.dev.reset() } } @@ -598,18 +604,6 @@ where M: MsiSender, D: Device, { - fn disable_all_irqs(&self) -> Result<()> { - let vfio_irq_disable_all = VfioIrqSet { - argsz: size_of::>() as u32, - flags: VfioIrqSetFlag::DATA_NONE | VfioIrqSetFlag::ACTION_TRIGGER, - index: VfioPciIrq::MSIX.raw(), - start: 0, - count: 0, - data: VfioIrqSetData { eventfds: [] }, - }; - self.cdev.dev.set_irqs(&vfio_irq_disable_all) - } - fn enable_irqfd(&self, index: usize) -> Result<()> { let mut entries = self.table.entries.write(); let Some(entry) = entries.get_mut(index) else { @@ -642,7 +636,7 @@ where // subindex for the first time. // As long as the following set_irqs() succeeds, we can safely ignore // the error here. - let _ = self.disable_all_irqs(); + let _ = self.cdev.dev.disable_all_irqs(VfioPciIrq::MSIX); let mut eventfds = [-1; 2048]; let mut count = 0; @@ -712,3 +706,125 @@ where Ok(Action::None) } } + +#[derive(Debug, Default, Clone, FromBytes, Immutable, IntoBytes, Layout)] +#[repr(C)] +struct MsiCapBody { + data: [u32; 4], +} +impl_mmio_for_zerocopy!(MsiCapBody); + +#[derive(Debug)] +struct MsiCapMmio +where + M: MsiSender, +{ + cap: RwLock<(MsiCapHdr, MsiCapBody)>, + dev: Arc>, + irqfds: Box<[M::IrqFd]>, +} + +impl MsiCapMmio +where + M: MsiSender, + D: Device, +{ + fn update_msi(&self, ctrl: MsiMsgCtrl, data: &[u32; 4]) -> Result<()> { + let msg_mask = if ctrl.ext_msg_data() { + u32::MAX + } else { + 0xffff + }; + let (addr, msg) = if ctrl.addr_64_cap() { + ((data[1] as u64) << 32 | data[0] as u64, data[2] & msg_mask) + } else { + (data[0] as u64, data[1] & msg_mask) + }; + let mask = match (ctrl.addr_64_cap(), ctrl.per_vector_masking_cap()) { + (true, true) => data[3], + (false, true) => data[2], + (_, false) => 0, + }; + let count = 1 << ctrl.multi_msg(); + for (index, irqfd) in self.irqfds.iter().enumerate() { + irqfd.set_masked(true)?; + if !ctrl.enable() || index >= count || mask & (1 << index) > 0 { + continue; + } + let msg = msg | index as u32; + irqfd.set_addr_hi((addr >> 32) as u32)?; + irqfd.set_addr_lo(addr as u32)?; + irqfd.set_data(msg)?; + irqfd.set_masked(false)?; + } + Ok(()) + } +} + +impl Mmio for MsiCapMmio +where + D: Device, + M: MsiSender, +{ + fn size(&self) -> u64 { + let (hdr, _) = &*self.cap.read(); + hdr.control.cap_size() as u64 + } + + fn read(&self, offset: u64, size: u8) -> mem::Result { + let (hdr, body) = &*self.cap.read(); + let ctrl = hdr.control; + match offset { + 0..4 => hdr.read(offset, size), + 0x10 if ctrl.per_vector_masking_cap() && !ctrl.addr_64_cap() => Ok(0), + 0x14 if ctrl.per_vector_masking_cap() && ctrl.addr_64_cap() => Ok(0), + _ => body.read(offset - size_of_val(hdr) as u64, size), + } + } + + fn write(&self, offset: u64, size: u8, val: u64) -> mem::Result { + let (hdr, body) = &mut *self.cap.write(); + let mut need_update = false; + match (offset as usize, size) { + (0x2, 2) => { + let ctrl = &mut hdr.control; + let new_ctrl = MsiMsgCtrl(val as u16); + if !ctrl.enable() || !new_ctrl.enable() { + let multi_msg = min(ctrl.multi_msg_cap(), new_ctrl.multi_msg()); + ctrl.set_multi_msg(multi_msg); + } + need_update = ctrl.enable() != new_ctrl.enable() + || (new_ctrl.enable() && ctrl.ext_msg_data() != new_ctrl.ext_msg_data()); + ctrl.set_ext_msg_data(new_ctrl.ext_msg_data()); + ctrl.set_enable(new_ctrl.enable()); + } + (0x4 | 0x8 | 0xc | 0x10, 2 | 4) => { + let data_offset = (offset as usize - size_of_val(hdr)) >> 2; + let reg = &mut body.data[data_offset]; + need_update = hdr.control.enable() && *reg != val as u32; + *reg = val as u32; + } + _ => log::error!( + "{}: write 0x{val:0width$x} to invalid offset 0x{offset:x}.", + self.dev.name, + width = 2 * size as usize + ), + } + if need_update { + self.update_msi(hdr.control, &body.data) + .map_err(boxed_debug_trace) + .context(mem::error::Mmio)?; + } + Ok(Action::None) + } +} + +impl PciConfigArea for MsiCapMmio +where + D: Device, + M: MsiSender, +{ + fn reset(&self) { + log::error!("{}: TODO: MsiCapMmio reset.", self.dev.name) + } +} diff --git a/alioth/src/virtio/pci.rs b/alioth/src/virtio/pci.rs index 219c82f3..d6267d7f 100644 --- a/alioth/src/virtio/pci.rs +++ b/alioth/src/virtio/pci.rs @@ -31,8 +31,8 @@ use crate::pci::cap::{ MsixTableMmioEntry, PciCap, PciCapHdr, PciCapId, PciCapList, }; use crate::pci::config::{ - CommonHeader, DeviceHeader, EmulatedConfig, HeaderType, PciConfig, BAR_MEM32, BAR_MEM64, - BAR_PREFETCHABLE, + CommonHeader, DeviceHeader, EmulatedConfig, HeaderType, PciConfig, PciConfigArea, BAR_MEM32, + BAR_MEM64, BAR_PREFETCHABLE, }; use crate::pci::{self, Pci, PciBar}; use crate::utils::{ @@ -588,12 +588,14 @@ pub struct VirtioPciCap { } impl_mmio_for_zerocopy!(VirtioPciCap); +impl PciConfigArea for VirtioPciCap { + fn reset(&self) {} +} + impl PciCap for VirtioPciCap { fn set_next(&mut self, val: u8) { self.header.next = val } - - fn reset(&self) {} } #[repr(C, align(4))] @@ -605,11 +607,14 @@ pub struct VirtioPciCap64 { } impl_mmio_for_zerocopy!(VirtioPciCap64); +impl PciConfigArea for VirtioPciCap64 { + fn reset(&self) {} +} + impl PciCap for VirtioPciCap64 { fn set_next(&mut self, val: u8) { PciCap::set_next(&mut self.cap, val) } - fn reset(&self) {} } #[repr(C, align(4))] @@ -620,11 +625,14 @@ pub struct VirtioPciNotifyCap { } impl_mmio_for_zerocopy!(VirtioPciNotifyCap); +impl PciConfigArea for VirtioPciNotifyCap { + fn reset(&self) {} +} + impl PciCap for VirtioPciNotifyCap { fn set_next(&mut self, val: u8) { self.cap.header.next = val; } - fn reset(&self) {} } #[derive(Debug)]