diff --git a/aarch64/src/lib.rs b/aarch64/src/lib.rs index e14e13e8d0..912fa04d91 100644 --- a/aarch64/src/lib.rs +++ b/aarch64/src/lib.rs @@ -112,7 +112,7 @@ const AARCH64_PROTECTED_VM_FW_START: u64 = AARCH64_PHYS_MEM_START - AARCH64_PROTECTED_VM_FW_MAX_SIZE; const AARCH64_PVTIME_IPA_MAX_SIZE: u64 = 0x10000; -const AARCH64_PVTIME_IPA_START: u64 = AARCH64_MMIO_BASE - AARCH64_PVTIME_IPA_MAX_SIZE; +const AARCH64_PVTIME_IPA_START: u64 = 0x1ff0000; const AARCH64_PVTIME_SIZE: u64 = 64; // These constants indicate the placement of the GIC registers in the physical @@ -154,10 +154,10 @@ const AARCH64_VMWDT_SIZE: u64 = 0x1000; const AARCH64_PCI_CAM_BASE_DEFAULT: u64 = 0x10000; // Default PCI MMIO configuration region size. const AARCH64_PCI_CAM_SIZE_DEFAULT: u64 = 0x1000000; -// This is the base address of MMIO devices. -const AARCH64_MMIO_BASE: u64 = 0x2000000; -// Size of the whole MMIO region. -const AARCH64_MMIO_SIZE: u64 = 0x2000000; +// Default PCI mem base address. +const AARCH64_PCI_MEM_BASE_DEFAULT: u64 = 0x2000000; +// Default PCI mem size. +const AARCH64_PCI_MEM_SIZE_DEFAULT: u64 = 0x2000000; // Virtio devices start at SPI interrupt number 4 const AARCH64_IRQ_BASE: u32 = 4; @@ -236,6 +236,8 @@ pub enum Error { Cmdline(kernel_cmdline::Error), #[error("bad PCI CAM configuration: {0}")] ConfigurePciCam(String), + #[error("bad PCI mem configuration: {0}")] + ConfigurePciMem(String), #[error("failed to configure CPU Frequencies: {0}")] CpuFrequencies(base::Error), #[error("failed to configure CPU topology: {0}")] @@ -384,6 +386,7 @@ fn main_memory_size(components: &VmComponents, hypervisor: &(impl Hypervisor + ? pub struct ArchMemoryLayout { pci_cam: AddressRange, + pci_mem: AddressRange, } impl arch::LinuxArch for AArch64 { @@ -414,7 +417,20 @@ impl arch::LinuxArch for AArch64 { ))); } - Ok(ArchMemoryLayout { pci_cam }) + let pci_mem = match components.pci_config.mem { + Some(MemoryRegionConfig { start, size }) => AddressRange::from_start_and_size( + start, + size.unwrap_or(AARCH64_PCI_MEM_SIZE_DEFAULT), + ) + .ok_or(Error::ConfigurePciMem("region overflowed".to_string()))?, + None => AddressRange::from_start_and_size( + AARCH64_PCI_MEM_BASE_DEFAULT, + AARCH64_PCI_MEM_SIZE_DEFAULT, + ) + .unwrap(), + }; + + Ok(ArchMemoryLayout { pci_cam, pci_mem }) } /// Returns a Vec of the valid memory addresses. @@ -456,7 +472,7 @@ impl arch::LinuxArch for AArch64 { fn get_system_allocator_config( vm: &V, - _arch_memory_layout: &Self::ArchMemoryLayout, + arch_memory_layout: &Self::ArchMemoryLayout, ) -> SystemAllocatorConfig { let guest_phys_end = 1u64 << vm.get_guest_phys_addr_bits(); // The platform MMIO region is immediately past the end of RAM. @@ -474,8 +490,7 @@ impl arch::LinuxArch for AArch64 { }); SystemAllocatorConfig { io: None, - low_mmio: AddressRange::from_start_and_size(AARCH64_MMIO_BASE, AARCH64_MMIO_SIZE) - .expect("invalid mmio region"), + low_mmio: arch_memory_layout.pci_mem, high_mmio: AddressRange::from_start_and_size(high_mmio_base, high_mmio_size) .expect("invalid high mmio region"), platform_mmio: Some( diff --git a/arch/src/lib.rs b/arch/src/lib.rs index 9d042fe0fd..938ca0ba1a 100644 --- a/arch/src/lib.rs +++ b/arch/src/lib.rs @@ -344,7 +344,6 @@ pub enum VcpuAffinity { } /// Memory region with optional size. -#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, FromKeyValues)] pub struct MemoryRegionConfig { pub start: u64, @@ -357,6 +356,8 @@ pub struct PciConfig { /// region for PCI Configuration Access Mechanism #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] pub cam: Option, + /// region for non-prefetchable PCI device memory below 4G + pub mem: Option, } /// Holds the pieces needed to build a VM. Passed to `build_vm` in the `LinuxArch` trait below to @@ -401,8 +402,6 @@ pub struct VmComponents { pub normalized_cpu_capacities: BTreeMap, pub pci_config: PciConfig, #[cfg(target_arch = "x86_64")] - pub pci_low_start: Option, - #[cfg(target_arch = "x86_64")] pub pcie_ecam: Option, pub pflash_block_size: u32, pub pflash_image: Option, diff --git a/docs/book/src/appendix/memory_layout.md b/docs/book/src/appendix/memory_layout.md index f7311fbf1b..33b82f0d63 100644 --- a/docs/book/src/appendix/memory_layout.md +++ b/docs/book/src/appendix/memory_layout.md @@ -64,8 +64,8 @@ These apply for all boot modes. | [`AARCH64_VMWDT_ADDR`] | `3000` | `4000` | 4 KiB | Watchdog device | | [`AARCH64_PCI_CAM_BASE_DEFAULT`] | `1_0000` | `101_0000` | 16 MiB | PCI configuration (CAM) | | [`AARCH64_VIRTFREQ_BASE`] | `104_0000` | `105_0000` | 64 KiB | Virtual cpufreq device | -| [`AARCH64_PVTIME_IPA_START`] | `1f0_0000` | `200_0000` | 64 KiB | Paravirtualized time | -| [`AARCH64_MMIO_BASE`] | `200_0000` | `400_0000` | 32 MiB | Low MMIO allocation area | +| [`AARCH64_PVTIME_IPA_START`] | `1ff_0000` | `200_0000` | 64 KiB | Paravirtualized time | +| [`AARCH64_PCI_CAM_BASE_DEFAULT`] | `200_0000` | `400_0000` | 32 MiB | Low MMIO allocation area | | [`AARCH64_GIC_CPUI_BASE`] | `3ffd_0000` | `3fff_0000` | 128 KiB | vGIC | | [`AARCH64_GIC_DIST_BASE`] | `3fff_0000` | `4000_0000` | 64 KiB | vGIC | | [`AARCH64_PROTECTED_VM_FW_START`] | `7fc0_0000` | `8000_0000` | 4 MiB | pVM firmware (if running a protected VM) | @@ -107,7 +107,7 @@ with a 16 MiB alignment. [`aarch64_vmwdt_addr`]: https://crsrc.org/o/src/platform/crosvm/aarch64/src/lib.rs;l=187?q=AARCH64_VMWDT_ADDR [`aarch64_pci_cfg_base`]: https://crsrc.org/o/src/platform/crosvm/aarch64/src/lib.rs;l=192?q=AARCH64_PCI_CAM_BASE_DEFAULT [`aarch64_virtfreq_base`]: https://crsrc.org/o/src/platform/crosvm/aarch64/src/lib.rs;l=207?q=AARCH64_VIRTFREQ_BASE -[`aarch64_mmio_base`]: https://crsrc.org/o/src/platform/crosvm/aarch64/src/lib.rs;l=196?q=AARCH64_MMIO_BASE +[`aarch64_mmio_base`]: https://crsrc.org/o/src/platform/crosvm/aarch64/src/lib.rs;l=196?q=AARCH64_PCI_CAM_BASE_DEFAULT [`aarch64_gic_cpui_base`]: https://crsrc.org/o/src/platform/crosvm/devices/src/irqchip/kvm/aarch64.rs;l=106?q=AARCH64_GIC_CPUI_BASE [`aarch64_gic_dist_base`]: https://crsrc.org/o/src/platform/crosvm/aarch64/src/lib.rs;l=105?q=AARCH64_GIC_DIST_BASE [`aarch64_pvtime_ipa_start`]: https://crsrc.org/o/src/platform/crosvm/aarch64/src/lib.rs;l=100?q=AARCH64_PVTIME_IPA_START diff --git a/riscv64/src/lib.rs b/riscv64/src/lib.rs index 68b9bb1196..90d1a27996 100644 --- a/riscv64/src/lib.rs +++ b/riscv64/src/lib.rs @@ -134,7 +134,9 @@ pub enum Error { InitrdLoadFailure(arch::LoadImageError), #[error("kernel could not be loaded: {0}")] KernelLoadFailure(arch::LoadImageError), - #[error("protected vms not supported on riscv(yet)")] + #[error("PCI mem region not configurable on riscv (yet)")] + PciMemNotConfigurable, + #[error("protected vms not supported on riscv (yet)")] ProtectedVmUnsupported, #[error("ramoops address is different from high_mmio_base: {0} vs {1}")] RamoopsAddress(u64, u64), @@ -167,8 +169,11 @@ impl arch::LinuxArch for Riscv64 { type ArchMemoryLayout = ArchMemoryLayout; fn arch_memory_layout( - _components: &VmComponents, + components: &VmComponents, ) -> std::result::Result { + if components.pci_config.mem.is_some() { + return Err(Error::PciMemNotConfigurable); + } Ok(ArchMemoryLayout {}) } diff --git a/src/crosvm/cmdline.rs b/src/crosvm/cmdline.rs index e6db292516..6c6cfe5aad 100644 --- a/src/crosvm/cmdline.rs +++ b/src/crosvm/cmdline.rs @@ -24,6 +24,8 @@ use std::sync::atomic::Ordering; use arch::CpuSet; use arch::FdtPosition; +#[cfg(target_arch = "x86_64")] +use arch::MemoryRegionConfig; use arch::PciConfig; use arch::Pstore; #[cfg(target_arch = "x86_64")] @@ -1791,6 +1793,9 @@ pub struct RunCommand { #[merge(strategy = overwrite_option)] /// PCI parameters. /// + /// Possible key values: + /// mem=[start=INT,size=INT] - region for non-prefetchable PCI device memory below 4G + /// /// Possible key values (aarch64 only): /// cam=[start=INT,size=INT] - region for PCI Configuration Access Mechanism pub pci: Option, @@ -3617,11 +3622,21 @@ impl TryFrom for super::config::Config { cfg.enable_hwp = cmd.enable_hwp.unwrap_or_default(); cfg.force_s2idle = cmd.s2idle.unwrap_or_default(); cfg.pcie_ecam = cmd.pcie_ecam; - cfg.pci_low_start = cmd.pci_start; cfg.no_i8042 = cmd.no_i8042.unwrap_or_default(); cfg.no_rtc = cmd.no_rtc.unwrap_or_default(); cfg.smbios = cmd.smbios.unwrap_or_default(); + if let Some(pci_start) = cmd.pci_start { + if cfg.pci_config.mem.is_some() { + return Err("--pci-start cannot be used with --pci mem=[...]".to_string()); + } + log::warn!("`--pci-start` is deprecated; use `--pci mem=[start={pci_start:#?}]"); + cfg.pci_config.mem = Some(MemoryRegionConfig { + start: pci_start, + size: None, + }); + } + if !cmd.oem_strings.is_empty() { log::warn!( "`--oem-strings` is deprecated; use `--smbios oem-strings=[...]` instead." diff --git a/src/crosvm/config.rs b/src/crosvm/config.rs index 85fad5458f..bd8ded3149 100644 --- a/src/crosvm/config.rs +++ b/src/crosvm/config.rs @@ -843,8 +843,6 @@ pub struct Config { #[cfg(feature = "pci-hotplug")] pub pci_hotplug_slots: Option, #[cfg(target_arch = "x86_64")] - pub pci_low_start: Option, - #[cfg(target_arch = "x86_64")] pub pcie_ecam: Option, pub per_vm_core_scheduling: bool, pub pflash_parameters: Option, @@ -1082,8 +1080,6 @@ impl Default for Config { #[cfg(feature = "pci-hotplug")] pci_hotplug_slots: None, #[cfg(target_arch = "x86_64")] - pci_low_start: None, - #[cfg(target_arch = "x86_64")] pcie_ecam: None, per_vm_core_scheduling: false, pflash_parameters: None, @@ -1436,7 +1432,6 @@ mod tests { use super::*; - #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] fn config_from_args(args: &[&str]) -> Config { crate::crosvm::cmdline::RunCommand::from_args(&[], args) .unwrap() @@ -2521,6 +2516,7 @@ mod tests { start: 0x123, size: None, }), + ..PciConfig::default() } ); assert_eq!( @@ -2530,6 +2526,31 @@ mod tests { start: 0x123, size: Some(0x456), }), + ..PciConfig::default() + }, + ); + } + + #[test] + fn parse_pci_mem() { + assert_eq!( + config_from_args(&["--pci", "mem=[start=0x123]", "/dev/null"]).pci_config, + PciConfig { + mem: Some(arch::MemoryRegionConfig { + start: 0x123, + size: None, + }), + ..PciConfig::default() + } + ); + assert_eq!( + config_from_args(&["--pci", "mem=[start=0x123,size=0x456]", "/dev/null"]).pci_config, + PciConfig { + mem: Some(arch::MemoryRegionConfig { + start: 0x123, + size: Some(0x456), + }), + ..PciConfig::default() }, ); } diff --git a/src/crosvm/sys/linux.rs b/src/crosvm/sys/linux.rs index 8b80a680d2..313d9198f1 100644 --- a/src/crosvm/sys/linux.rs +++ b/src/crosvm/sys/linux.rs @@ -1501,8 +1501,6 @@ fn setup_vm_components(cfg: &Config) -> Result { pci_config: cfg.pci_config, #[cfg(target_arch = "x86_64")] pcie_ecam: cfg.pcie_ecam, - #[cfg(target_arch = "x86_64")] - pci_low_start: cfg.pci_low_start, dynamic_power_coefficient: cfg.dynamic_power_coefficient.clone(), boot_cpu: cfg.boot_cpu, #[cfg(any(target_arch = "arm", target_arch = "aarch64"))] diff --git a/src/sys/windows.rs b/src/sys/windows.rs index 77ec75c0f1..12d28ade07 100644 --- a/src/sys/windows.rs +++ b/src/sys/windows.rs @@ -2112,8 +2112,6 @@ fn setup_vm_components(cfg: &Config) -> Result { pvm_fw: None, pci_config: cfg.pci_config, #[cfg(target_arch = "x86_64")] - pci_low_start: cfg.pci_low_start, - #[cfg(target_arch = "x86_64")] pcie_ecam: cfg.pcie_ecam, #[cfg(target_arch = "x86_64")] smbios: cfg.smbios.clone(), diff --git a/x86_64/src/lib.rs b/x86_64/src/lib.rs index c06d0fbcdb..e4d8af7c8f 100644 --- a/x86_64/src/lib.rs +++ b/x86_64/src/lib.rs @@ -60,6 +60,8 @@ use arch::CpuSet; use arch::DtbOverlay; use arch::FdtPosition; use arch::GetSerialCmdlineError; +use arch::MemoryRegionConfig; +use arch::PciConfig; use arch::RunnableLinuxVm; use arch::VmComponents; use arch::VmImage; @@ -172,6 +174,8 @@ pub enum Error { CommandLineOverflow, #[error("failed to configure hotplugged pci device: {0}")] ConfigurePciDevice(arch::DeviceRegistrationError), + #[error("bad PCI mem configuration: {0}")] + ConfigurePciMem(String), #[error("failed to configure segment registers: {0}")] ConfigureSegments(regs::Error), #[error("error configuring the system")] @@ -355,7 +359,7 @@ const MEM_32BIT_GAP_SIZE: u64 = 768 * MB; const END_ADDR_BEFORE_32BITS: u64 = FIRST_ADDR_PAST_32BITS - MEM_32BIT_GAP_SIZE; // Reserved memory for nand_bios/LAPIC/IOAPIC/HPET/..... const RESERVED_MEM_SIZE: u64 = 0x800_0000; -const PCI_MMIO_END: u64 = FIRST_ADDR_PAST_32BITS - RESERVED_MEM_SIZE - 1; +const DEFAULT_PCI_MEM_END: u64 = FIRST_ADDR_PAST_32BITS - RESERVED_MEM_SIZE - 1; // Reserve 64MB for pcie enhanced configuration const DEFAULT_PCIE_CFG_MMIO_SIZE: u64 = 0x400_0000; const DEFAULT_PCIE_CFG_MMIO_END: u64 = FIRST_ADDR_PAST_32BITS - RESERVED_MEM_SIZE - 1; @@ -413,8 +417,8 @@ pub struct ArchMemoryLayout { } pub fn create_arch_memory_layout( + pci_config: &PciConfig, pcie_ecam: Option, - pci_low_start: Option, has_protected_vm_firmware: bool, ) -> Result { const DEFAULT_PCIE_CFG_MMIO: AddressRange = AddressRange { @@ -424,18 +428,21 @@ pub fn create_arch_memory_layout( let pcie_cfg_mmio = pcie_ecam.unwrap_or(DEFAULT_PCIE_CFG_MMIO); - let pci_mmio_before_32bit = if let Some(pci_low) = pci_low_start { - AddressRange { - start: pci_low, - end: PCI_MMIO_END, + let pci_mmio_before_32bit = match pci_config.mem { + Some(MemoryRegionConfig { + start, + size: Some(size), + }) => AddressRange::from_start_and_size(start, size) + .ok_or(Error::ConfigurePciMem("region overflowed".to_string()))?, + Some(MemoryRegionConfig { start, size: None }) => { + AddressRange::from_start_and_end(start, DEFAULT_PCI_MEM_END) } - } else { - AddressRange { - start: pcie_cfg_mmio + None => AddressRange::from_start_and_end( + pcie_cfg_mmio .start .min(FIRST_ADDR_PAST_32BITS - MEM_32BIT_GAP_SIZE), - end: PCI_MMIO_END, - } + DEFAULT_PCI_MEM_END, + ), }; let pvmfw_mem = if has_protected_vm_firmware { @@ -753,8 +760,8 @@ impl arch::LinuxArch for X8664arch { components: &VmComponents, ) -> std::result::Result { create_arch_memory_layout( + &components.pci_config, components.pcie_ecam, - components.pci_low_start, components.hv_cfg.protection_type.runs_firmware(), ) } @@ -2363,9 +2370,14 @@ mod tests { const TEST_MEMORY_SIZE: u64 = 2 * GB; fn setup() -> ArchMemoryLayout { + let pci_config = PciConfig { + mem: Some(MemoryRegionConfig { + start: 2 * GB, + size: None, + }), + }; let pcie_ecam = Some(AddressRange::from_start_and_size(3 * GB, 256 * MB).unwrap()); - let pci_start = Some(2 * GB); - create_arch_memory_layout(pcie_ecam, pci_start, false).unwrap() + create_arch_memory_layout(&pci_config, pcie_ecam, false).unwrap() } #[test] @@ -2487,7 +2499,7 @@ mod tests { #[test] fn check_32bit_gap_size_alignment() { let arch_memory_layout = setup(); - // pci_low_start is 256 MB aligned to be friendly for MTRR mappings. + // pci_mmio_before_32bit is 256 MB aligned to be friendly for MTRR mappings. assert_eq!( arch_memory_layout.pci_mmio_before_32bit.start % (256 * MB), 0