mirrors/crosvm
1
0
Fork 0
mirror of https://chromium.googlesource.com/crosvm/crosvm synced 2024-11-28 09:33:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

pcie: Link virtual pcie root port to physical pcie root port

Browse source 
On ManaTEE, each physical pcie root port has a virtual pcie root port in
CrOS, this commit links virtual RP to physical RP. The following virtual
RP's config registers are from physical:
--devie ID
--device type
--bridge bus range

BUG=b:185084350
TEST=Check virtual pcie RP config registers in ManaTEE CrOS

Change-Id: Ia38a0f2417331d562dbb305433c68a3bee678bde
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/3423459
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Daniel Verkamp <dverkamp@chromium.org>
Commit-Queue: Daniel Verkamp <dverkamp@chromium.org>
This commit is contained in:
Xiong Zhang 2021-12-23 16:32:20 +08:00 committed by Commit Bot
parent 10e4637525
commit cf093620a8
5 changed files with 244 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
devices/src/pci/pci_configuration.rs
View file

@ -19,7 +19,7 @@ pub const COMMAND_REG: usize = 1;
pub const COMMAND_REG_IO_SPACE_MASK: u32 = 0x0000_0001;
pub const COMMAND_REG_MEMORY_SPACE_MASK: u32 = 0x0000_0002;
const STATUS_REG: usize = 1;
const STATUS_REG_CAPABILITIES_USED_MASK: u32 = 0x0010_0000;
pub const STATUS_REG_CAPABILITIES_USED_MASK: u32 = 0x0010_0000;
pub const HEADER_TYPE_REG: usize = 3;
pub const HEADER_TYPE_MULTIFUNCTION_MASK: u32 = 0x0080_0000;
pub const BAR0_REG: usize = 4;
@ -31,7 +31,8 @@ const BAR_ROM_MIN_SIZE: u64 = 2048;
pub const NUM_BAR_REGS: usize = 7; // 6 normal BARs + expansion ROM BAR.
pub const ROM_BAR_IDX: PciBarIndex = 6;
pub const ROM_BAR_REG: usize = 12;
const CAPABILITY_LIST_HEAD_OFFSET: usize = 0x34;
pub const CAPABILITY_LIST_HEAD_OFFSET: usize = 0x34;
pub const PCI_CAP_NEXT_POINTER: usize = 0x1;
const FIRST_CAPABILITY_OFFSET: usize = 0x40;
const CAPABILITY_MAX_OFFSET: usize = 255;

1
devices/src/pci/pcie/mod.rs
View file

@ -4,6 +4,7 @@
mod pci_bridge;
mod pcie_device;
mod pcie_host;
mod pcie_rp;
pub use pci_bridge::PciBridge;

2
devices/src/pci/pcie/pci_bridge.rs
View file

@ -21,7 +21,7 @@ use crate::pci::pcie::pcie_device::PcieDevice;
const BR_MSIX_TABLE_OFFSET: u64 = 0x0;
const BR_MSIX_PBA_OFFSET: u64 = 0x100;
const PCI_BRIDGE_BAR_SIZE: u64 = 0x1000;
const BR_BUS_NUMBER_REG: usize = 0x6;
pub const BR_BUS_NUMBER_REG: usize = 0x6;
const BR_MEM_REG: usize = 0x8;
const BR_PREF_MEM_LOW_REG: usize = 0x9;
const BR_PREF_MEM_BASE_HIGH_REG: usize = 0xa;

162
devices/src/pci/pcie/pcie_host.rs Normal file
View file

@ -0,0 +1,162 @@
// Copyright 2021 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use std::fs::{File, OpenOptions};
use std::os::unix::fs::FileExt;
use std::path::{Path, PathBuf};
use anyhow::{anyhow, Context, Result};
use base::error;
use data_model::DataInit;
use crate::pci::{PciCapabilityID, PciClassCode};
use crate::pci::pci_configuration::{
PciBridgeSubclass, CAPABILITY_LIST_HEAD_OFFSET, PCI_CAP_NEXT_POINTER,
};
use crate::pci::pcie::pci_bridge::{PciBridgeBusRange, BR_BUS_NUMBER_REG};
use crate::pci::pcie::*;
// Host Pci device's sysfs config file
struct PciHostConfig {
config_file: File,
}
impl PciHostConfig {
// Create a new host pci device's sysfs config file
fn new(host_sysfs_path: &Path) -> Result<Self> {
let mut config_path = PathBuf::new();
config_path.push(host_sysfs_path);
config_path.push("config");
let f = OpenOptions::new()
.write(true)
.read(true)
.open(config_path.as_path())
.with_context(|| format!("failed to open: {}", config_path.display()))?;
Ok(PciHostConfig { config_file: f })
}
// Read host pci device's config register
fn read_config<T: DataInit>(&self, offset: u64) -> T {
let length = std::mem::size_of::<T>();
let mut buf = vec![0u8; length];
if offset % length as u64 != 0 {
error!(
"read_config, offset {} isn't aligned to length {}",
offset, length
);
} else if let Err(e) = self.config_file.read_exact_at(&mut buf, offset) {
error!("failed to read host sysfs config: {}", e);
}
T::from_slice(&buf)
.copied()
.expect("failed to convert host sysfs config data from slice")
}
// write host pci device's config register
#[allow(dead_code)]
fn write_config(&self, offset: u64, data: &[u8]) {
if offset % data.len() as u64 != 0 {
error!(
"write_config, offset {} isn't aligned to length {}",
offset,
data.len()
);
return;
}
if let Err(e) = self.config_file.write_all_at(data, offset) {
error!("failed to write host sysfs config: {}", e);
}
}
}
const PCI_CONFIG_DEVICE_ID: u64 = 0x02;
const PCI_BASE_CLASS_CODE: u64 = 0x0B;
const PCI_SUB_CLASS_CODE: u64 = 0x0A;
/// Pcie root port device has a corresponding host pcie root port.
pub struct PcieHostRootPort {
host_config: PciHostConfig,
host_name: String,
}
impl PcieHostRootPort {
pub fn new(host_sysfs_path: &Path) -> Result<Self> {
let host_config = PciHostConfig::new(host_sysfs_path)?;
let host_name = host_sysfs_path
.file_name()
.unwrap()
.to_str()
.unwrap()
.to_owned();
let base_class: u8 = host_config.read_config(PCI_BASE_CLASS_CODE);
if base_class != PciClassCode::BridgeDevice.get_register_value() {
return Err(anyhow!("host {} isn't bridge", host_name));
}
let sub_class: u8 = host_config.read_config(PCI_SUB_CLASS_CODE);
if sub_class != PciBridgeSubclass::PciToPciBridge as u8 {
return Err(anyhow!("host {} isn't pci to pci bridge", host_name));
}
let mut pcie_cap_reg: u8 = 0;
let mut cap_next: u8 = host_config.read_config(CAPABILITY_LIST_HEAD_OFFSET as u64);
let mut counter: u16 = 0;
while cap_next != 0 && counter < 256 {
let cap_id: u8 = host_config.read_config(cap_next.into());
if cap_id == PciCapabilityID::PciExpress as u8 {
pcie_cap_reg = cap_next;
break;
}
let offset = cap_next as u64 + PCI_CAP_NEXT_POINTER as u64;
cap_next = host_config.read_config(offset);
counter += 1;
}
if pcie_cap_reg == 0 {
return Err(anyhow!("host {} isn't pcie device", host_name));
}
let device_cap: u8 = host_config.read_config(pcie_cap_reg as u64 + PCIE_CAP_VERSION as u64);
if (device_cap >> PCIE_TYPE_SHIFT) != PcieDevicePortType::RootPort as u8 {
return Err(anyhow!("host {} isn't pcie root port", host_name));
}
Ok(PcieHostRootPort {
host_config,
host_name,
})
}
pub fn get_bus_range(&self) -> PciBridgeBusRange {
let bus_num: u32 = self.host_config.read_config((BR_BUS_NUMBER_REG * 4) as u64);
let primary = (bus_num & 0xFF) as u8;
let secondary = ((bus_num >> 8) & 0xFF) as u8;
let subordinate = ((bus_num >> 16) & 0xFF) as u8;
PciBridgeBusRange {
primary,
secondary,
subordinate,
}
}
pub fn read_device_id(&self) -> u16 {
self.host_config.read_config::<u16>(PCI_CONFIG_DEVICE_ID)
}
pub fn host_name(&self) -> String {
self.host_name.clone()
}
pub fn read_config(&self, reg_idx: usize, data: &mut u32) {
if reg_idx == 3 {
// device header type
*data = self.host_config.read_config(0x0C);
}
}
pub fn write_config(&mut self, _reg_idx: usize, _offset: u64, _data: &[u8]) {}
}

86
devices/src/pci/pcie/pcie_rp.rs
View file

@ -1,6 +1,7 @@
// Copyright 2021 The Chromium OS Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
use std::path::Path;
use std::sync::Arc;
use sync::Mutex;
@ -10,7 +11,10 @@ use crate::pci::{MsixConfig, PciAddress, PciCapability, PciDeviceError};
use crate::pci::pcie::pci_bridge::PciBridgeBusRange;
use crate::pci::pcie::pcie_device::{PcieCap, PcieDevice};
use crate::pci::pcie::pcie_host::PcieHostRootPort;
use crate::pci::pcie::*;
use anyhow::{anyhow, Result};
use base::warn;
use data_model::DataInit;
use resources::{Alloc, SystemAllocator};
@ -30,6 +34,7 @@ pub struct PcieRootPort {
bus_range: PciBridgeBusRange,
downstream_device: Option<(PciAddress, Option<HostHotPlugKey>)>,
removed_downstream: Option<PciAddress>,
pcie_host: Option<PcieHostRootPort>,
}
impl PcieRootPort {
@ -49,9 +54,37 @@ impl PcieRootPort {
bus_range,
downstream_device: None,
removed_downstream: None,
pcie_host: None,
}
}
/// Constructs a new PCIE root port which associated with the host physical pcie RP
/// As a lot of checking is done on host pcie RP, if the check is failure, this
/// physical pcie RP will be ignored, and virtual pcie RP won't be created.
pub fn new_from_host(pcie_host_sysfs: &Path) -> Result<Self> {
let pcie_host = PcieHostRootPort::new(pcie_host_sysfs)?;
let bus_range = pcie_host.get_bus_range();
// if physical pcie root port isn't on bus 0, ignore this physical pcie root port.
if bus_range.primary != 0 {
return Err(anyhow!(
"physical pcie RP isn't on bus 0: {}",
bus_range.primary
));
}
Ok(PcieRootPort {
pcie_cap_reg_idx: None,
msix_config: None,
pci_address: None,
slot_control: PCIE_SLTCTL_PIC_OFF | PCIE_SLTCTL_AIC_OFF,
slot_status: 0,
bus_range,
downstream_device: None,
removed_downstream: None,
pcie_host: Some(pcie_host),
})
}
fn read_pcie_cap(&self, offset: usize, data: &mut u32) {
if offset == PCIE_SLTCTL_OFFSET {
*data = ((self.slot_status as u32) << 16) | (self.slot_control as u32);
@ -143,10 +176,17 @@ impl PcieRootPort {
impl PcieDevice for PcieRootPort {
fn get_device_id(&self) -> u16 {
PCIE_RP_DID
match &self.pcie_host {
Some(host) => host.read_device_id(),
None => PCIE_RP_DID,
}
}
fn debug_label(&self) -> String {
"PcieRootPort".to_string()
match &self.pcie_host {
Some(host) => host.host_name(),
None => "PcieRootPort".to_string(),
}
}
fn allocate_address(
@ -154,16 +194,33 @@ impl PcieDevice for PcieRootPort {
resources: &mut SystemAllocator,
) -> std::result::Result<PciAddress, PciDeviceError> {
if self.pci_address.is_none() {
self.pci_address =
match resources.allocate_pci(self.bus_range.primary, self.debug_label()) {
match &self.pcie_host {
Some(host) => {
let address = PciAddress::from_string(&host.host_name());
if resources.reserve_pci(
Alloc::PciBar {
bus: address.bus,
dev: address.dev,
func: address.func,
bar: 0,
},
host.host_name(),
) {
self.pci_address = Some(address);
} else {
self.pci_address = None;
}
}
None => match resources.allocate_pci(self.bus_range.primary, self.debug_label()) {
Some(Alloc::PciBar {
bus,
dev,
func,
bar: _,
}) => Some(PciAddress { bus, dev, func }),
_ => None,
}
}) => self.pci_address = Some(PciAddress { bus, dev, func }),
_ => self.pci_address = None,
},
}
}
self.pci_address.ok_or(PciDeviceError::PciAllocationFailed)
}
@ -193,6 +250,10 @@ impl PcieDevice for PcieRootPort {
self.read_pcie_cap(offset, data);
}
}
if let Some(host) = &self.pcie_host {
// pcie host may override some config registers
host.read_config(reg_idx, data);
}
}
fn write_config(&mut self, reg_idx: usize, offset: u64, data: &[u8]) {
@ -202,6 +263,10 @@ impl PcieDevice for PcieRootPort {
self.write_pcie_cap(delta, data);
}
}
if let Some(host) = self.pcie_host.as_mut() {
// device may write data to host, or do something at specific register write
host.write_config(reg_idx, offset, data);
}
}
fn get_bus_range(&self) -> Option<PciBridgeBusRange> {
@ -251,8 +316,11 @@ impl HotPlugBus for PcieRootPort {
self.trigger_hp_interrupt();
}
fn is_match(&self, _host_addr: PciAddress) -> Option<u8> {
if self.downstream_device.is_none() {
fn is_match(&self, host_addr: PciAddress) -> Option<u8> {
if self.downstream_device.is_none()
&& host_addr.bus >= self.bus_range.secondary
&& host_addr.bus <= self.bus_range.subordinate
{
Some(self.bus_range.secondary)
} else {
None