mirrors/alioth
1
0
Fork 0
mirror of https://github.com/google/alioth.git synced 2024-11-28 09:26:21 +00:00
Code Issues Projects Releases Packages Wiki Activity

refactor(kvm): separate out arch-dependent code

Browse source 
Signed-off-by: Changyuan Lyu <changyuanl@google.com>
This commit is contained in:
Changyuan Lyu 2024-06-16 11:23:19 -07:00 committed by Lencerf
parent c8e891e7e7
commit 6efa97ab84
4 changed files with 187 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

117
alioth/src/hv/kvm/kvm.rs
View file

@ -18,14 +18,17 @@ mod ioctls;
mod sev;
#[path = "vcpu/vcpu.rs"]
mod vcpu;
#[path = "vm/vm.rs"]
mod vm;
mod vmentry;
mod vmexit;
#[cfg(target_arch = "x86_64")]
mod x86_64;
use std::collections::HashMap;
use std::fs::File;
use std::mem::{size_of, transmute};
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
use std::os::fd::{FromRawFd, OwnedFd};
use std::path::{Path, PathBuf};
use std::ptr::null_mut;
use std::sync::atomic::AtomicU32;
@ -40,21 +43,13 @@ use snafu::{ResultExt, Snafu};
use crate::ffi;
#[cfg(target_arch = "x86_64")]
use crate::hv::Cpuid;
use crate::hv::{error, Coco, Hypervisor, MemMapOption, Result, VmConfig};
use crate::hv::{error, Hypervisor, MemMapOption, Result, VmConfig};
use bindings::{
KvmCap, KvmCpuid2, KvmCpuid2Flag, KvmCpuidEntry2, KvmCreateGuestMemfd, KvmEnableCap, KvmVmType,
KVM_API_VERSION, KVM_MAX_CPUID_ENTRIES,
};
use ioctls::{
kvm_check_extension, kvm_create_guest_memfd, kvm_create_irqchip, kvm_create_vm, kvm_enable_cap,
kvm_get_api_version, kvm_get_vcpu_mmap_size,
};
use bindings::{KvmCpuid2, KvmCpuid2Flag, KvmCpuidEntry2, KVM_API_VERSION, KVM_MAX_CPUID_ENTRIES};
#[cfg(target_arch = "x86_64")]
use ioctls::{kvm_get_supported_cpuid, kvm_set_identity_map_addr, kvm_set_tss_addr};
use ioctls::kvm_get_supported_cpuid;
use ioctls::{kvm_create_vm, kvm_get_api_version, kvm_get_vcpu_mmap_size};
use libc::SIGRTMIN;
use sev::bindings::{KvmSevInit, KVM_SEV_ES_INIT, KVM_SEV_INIT, KVM_SEV_INIT2};
use sev::SevFd;
use vm::{KvmVm, VmInner};
#[trace_error]
@ -143,87 +138,24 @@ impl Hypervisor for Kvm {
fn create_vm(&self, config: &VmConfig) -> Result<Self::Vm> {
let vcpu_mmap_size =
unsafe { kvm_get_vcpu_mmap_size(&self.fd) }.context(error::CreateVm)? as usize;
let kvm_vm_type = if let Some(Coco::AmdSnp { .. }) = &config.coco {
KvmVmType::SNP
} else {
KvmVmType::DEFAULT
};
let kvm_vm_type = Self::determine_vm_type(config)?;
let vm_fd = unsafe { kvm_create_vm(&self.fd, kvm_vm_type) }.context(error::CreateVm)?;
let fd = unsafe { OwnedFd::from_raw_fd(vm_fd) };
let sev_fd = if let Some(cv) = &config.coco {
match cv {
Coco::AmdSev { .. } | Coco::AmdSnp { .. } => Some(match &self.config.dev_sev {
Some(dev_sev) => SevFd::new(dev_sev),
None => SevFd::new("/dev/sev"),
}?),
}
} else {
None
};
let memfd = if let Some(Coco::AmdSnp { .. }) = &config.coco {
let mut request = KvmCreateGuestMemfd {
size: 1 << 48,
..Default::default()
};
let ret = unsafe { kvm_create_guest_memfd(&fd, &mut request) }
.context(kvm_error::GuestMemfd)?;
Some(unsafe { OwnedFd::from_raw_fd(ret) })
} else {
None
};
let kvm_vm_arch = self.create_vm_arch(config)?;
let memfd = self.create_guest_memfd(config, &fd)?;
let kvm_vm = KvmVm {
vm: Arc::new(VmInner {
fd,
sev_fd,
memfd,
ioeventfds: Mutex::new(HashMap::new()),
msi_table: RwLock::new(HashMap::new()),
next_msi_gsi: AtomicU32::new(0),
pin_map: AtomicU32::new(0),
arch: kvm_vm_arch,
}),
vcpu_mmap_size,
memory_created: false,
};
if kvm_vm.vm.sev_fd.is_some() {
match config.coco.as_ref().unwrap() {
Coco::AmdSev { policy } => {
if policy.es() {
kvm_vm.sev_op::<()>(KVM_SEV_ES_INIT, None)?;
} else {
kvm_vm.sev_op::<()>(KVM_SEV_INIT, None)?;
}
}
Coco::AmdSnp { .. } => {
let bitmap = unsafe { kvm_check_extension(&kvm_vm.vm, KvmCap::EXIT_HYPERCALL) }
.context(kvm_error::CheckExtension {
ext: "KVM_CAP_EXIT_HYPERCALL",
})?;
if bitmap != 0 {
let request = KvmEnableCap {
cap: KvmCap::EXIT_HYPERCALL,
args: [bitmap as _, 0, 0, 0],
flags: 0,
pad: [0; 64],
};
unsafe { kvm_enable_cap(&kvm_vm.vm, &request) }.context(
kvm_error::EnableCap {
cap: "KVM_CAP_EXIT_HYPERCALL",
},
)?;
}
let mut init = KvmSevInit::default();
kvm_vm.sev_op(KVM_SEV_INIT2, Some(&mut init))?;
log::debug!("vm-{}: snp init: {init:#x?}", kvm_vm.vm.as_raw_fd());
}
}
}
unsafe { kvm_create_irqchip(&kvm_vm.vm) }.context(error::CreateDevice)?;
// TODO should be in parameters
#[cfg(target_arch = "x86_64")]
unsafe { kvm_set_tss_addr(&kvm_vm.vm, 0xf000_0000) }.context(error::SetVmParam)?;
#[cfg(target_arch = "x86_64")]
unsafe { kvm_set_identity_map_addr(&kvm_vm.vm, &0xf000_3000) }
.context(error::SetVmParam)?;
self.vm_init_arch(config, &kvm_vm)?;
Ok(kvm_vm)
}
@ -253,26 +185,3 @@ impl Hypervisor for Kvm {
Ok(cpuids)
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
#[cfg(target_arch = "x86_64")]
fn test_get_supported_cpuid() {
let kvm = Kvm::new(KvmConfig::default()).unwrap();
let mut kvm_cpuid_exist = false;
let supported_cpuids = kvm.get_supported_cpuids().unwrap();
for cpuid in &supported_cpuids {
if cpuid.func == 0x4000_0000
&& cpuid.ebx.to_le_bytes() == *b"KVMK"
&& cpuid.ecx.to_le_bytes() == *b"VMKV"
&& cpuid.edx.to_le_bytes() == *b"M\0\0\0"
{
kvm_cpuid_exist = true;
}
}
assert!(kvm_cpuid_exist);
}
}

18
alioth/src/hv/kvm/vm.rs → alioth/src/hv/kvm/vm/vm.rs
View file

@ -12,6 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(target_arch = "x86_64")]
mod x86_64;
use std::collections::HashMap;
use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, OwnedFd, RawFd};
use std::os::unix::thread::JoinHandleExt;
@ -44,7 +47,6 @@ use crate::hv::kvm::sev::bindings::{
KVM_SEV_LAUNCH_UPDATE_VMSA, KVM_SEV_SNP_LAUNCH_FINISH, KVM_SEV_SNP_LAUNCH_START,
KVM_SEV_SNP_LAUNCH_UPDATE,
};
use crate::hv::kvm::sev::SevFd;
use crate::hv::kvm::vcpu::{KvmRunBlock, KvmVcpu};
use crate::hv::kvm::{kvm_error, KvmError};
use crate::hv::{
@ -52,21 +54,23 @@ use crate::hv::{
Vm, VmMemory,
};
#[cfg(target_arch = "x86_64")]
pub use x86_64::VmArch;
#[derive(Debug)]
pub(super) struct VmInner {
pub(super) fd: OwnedFd,
pub(super) sev_fd: Option<SevFd>,
pub(super) memfd: Option<OwnedFd>,
pub(super) ioeventfds: Mutex<HashMap<i32, KvmIoEventFd>>,
pub(super) msi_table: RwLock<HashMap<u32, KvmMsiEntryData>>,
pub(super) next_msi_gsi: AtomicU32,
pub(super) pin_map: AtomicU32,
pub(super) arch: VmArch,
}
impl VmInner {
fn update_routing_table(&self, table: &HashMap<u32, KvmMsiEntryData>) -> Result<(), KvmError> {
let mut entries = [KvmIrqRoutingEntry::default(); MAX_GSI_ROUTES];
let pin_map = self.pin_map.load(Ordering::Acquire);
let pin_map = self.arch.pin_map.load(Ordering::Acquire);
let mut index = 0;
for pin in 0..24 {
if pin_map & (1 << pin) == 0 {
@ -255,7 +259,7 @@ pub struct KvmIntxSender {
impl Drop for KvmIntxSender {
fn drop(&mut self) {
let pin_flag = 1 << (self.pin as u32);
self.vm.pin_map.fetch_and(!pin_flag, Ordering::AcqRel);
self.vm.arch.pin_map.fetch_and(!pin_flag, Ordering::AcqRel);
let request = KvmIrqfd {
fd: self.event_fd.as_raw_fd() as u32,
gsi: self.pin as u32,
@ -541,7 +545,7 @@ impl IoeventFdRegistry for KvmIoeventFdRegistry {
impl KvmVm {
pub(super) fn sev_op<T>(&self, cmd: u32, data: Option<&mut T>) -> Result<(), KvmError> {
let Some(sev_fd) = &self.vm.sev_fd else {
let Some(sev_fd) = &self.vm.arch.sev_fd else {
unreachable!("SevFd is not initialized")
};
let mut req = KvmSevCmd {
@ -594,7 +598,7 @@ impl Vm for KvmVm {
fn create_intx_sender(&self, pin: u8) -> Result<Self::IntxSender, Error> {
let pin_flag = 1 << pin;
if self.vm.pin_map.fetch_or(pin_flag, Ordering::AcqRel) & pin_flag == pin_flag {
if self.vm.arch.pin_map.fetch_or(pin_flag, Ordering::AcqRel) & pin_flag == pin_flag {
return Err(std::io::ErrorKind::AlreadyExists.into())
.context(error::CreateIntx { pin });
}

23
alioth/src/hv/kvm/vm/x86_64.rs Normal file
View file

@ -0,0 +1,23 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::atomic::AtomicU32;
use crate::hv::kvm::sev::SevFd;
#[derive(Debug)]
pub struct VmArch {
pub sev_fd: Option<SevFd>,
pub pin_map: AtomicU32,
}

140
alioth/src/hv/kvm/x86_64.rs Normal file
View file

@ -0,0 +1,140 @@
// Copyright 2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
use std::sync::atomic::AtomicU32;
use snafu::ResultExt;
use crate::hv::kvm::bindings::{KvmCap, KvmCreateGuestMemfd, KvmEnableCap, KvmVmType};
use crate::hv::kvm::ioctls::{
kvm_check_extension, kvm_create_guest_memfd, kvm_create_irqchip, kvm_enable_cap,
kvm_set_identity_map_addr, kvm_set_tss_addr,
};
use crate::hv::kvm::sev::bindings::{KvmSevInit, KVM_SEV_ES_INIT, KVM_SEV_INIT, KVM_SEV_INIT2};
use crate::hv::kvm::sev::SevFd;
use crate::hv::kvm::vm::{KvmVm, VmArch};
use crate::hv::kvm::{kvm_error, Kvm};
use crate::hv::{error, Coco, Result, VmConfig};
impl Kvm {
pub(super) fn determine_vm_type(config: &VmConfig) -> Result<KvmVmType> {
match &config.coco {
Some(Coco::AmdSnp { .. }) => Ok(KvmVmType::SNP),
_ => Ok(KvmVmType::DEFAULT),
}
}
pub(super) fn create_guest_memfd(
&self,
config: &VmConfig,
vm_fd: &OwnedFd,
) -> Result<Option<OwnedFd>> {
let memfd = if let Some(Coco::AmdSnp { .. }) = &config.coco {
let mut request = KvmCreateGuestMemfd {
size: 1 << 48,
..Default::default()
};
let ret = unsafe { kvm_create_guest_memfd(vm_fd, &mut request) }
.context(kvm_error::GuestMemfd)?;
Some(unsafe { OwnedFd::from_raw_fd(ret) })
} else {
None
};
Ok(memfd)
}
pub(super) fn create_vm_arch(&self, config: &VmConfig) -> Result<VmArch> {
let sev_fd = if let Some(cv) = &config.coco {
match cv {
Coco::AmdSev { .. } | Coco::AmdSnp { .. } => Some(match &self.config.dev_sev {
Some(dev_sev) => SevFd::new(dev_sev),
None => SevFd::new("/dev/sev"),
}?),
}
} else {
None
};
Ok(VmArch {
sev_fd,
pin_map: AtomicU32::new(0),
})
}
pub(super) fn vm_init_arch(&self, config: &VmConfig, kvm_vm: &KvmVm) -> Result<()> {
if kvm_vm.vm.arch.sev_fd.is_some() {
match config.coco.as_ref() {
Some(Coco::AmdSev { policy }) => {
if policy.es() {
kvm_vm.sev_op::<()>(KVM_SEV_ES_INIT, None)?;
} else {
kvm_vm.sev_op::<()>(KVM_SEV_INIT, None)?;
}
}
Some(Coco::AmdSnp { .. }) => {
let bitmap = unsafe { kvm_check_extension(&kvm_vm.vm, KvmCap::EXIT_HYPERCALL) }
.context(kvm_error::CheckExtension {
ext: "KVM_CAP_EXIT_HYPERCALL",
})?;
if bitmap != 0 {
let request = KvmEnableCap {
cap: KvmCap::EXIT_HYPERCALL,
args: [bitmap as _, 0, 0, 0],
flags: 0,
pad: [0; 64],
};
unsafe { kvm_enable_cap(&kvm_vm.vm, &request) }.context(
kvm_error::EnableCap {
cap: "KVM_CAP_EXIT_HYPERCALL",
},
)?;
}
let mut init = KvmSevInit::default();
kvm_vm.sev_op(KVM_SEV_INIT2, Some(&mut init))?;
log::debug!("vm-{}: snp init: {init:#x?}", kvm_vm.vm.as_raw_fd());
}
_ => {}
}
}
unsafe { kvm_create_irqchip(&kvm_vm.vm) }.context(error::CreateDevice)?;
// TODO should be in parameters
unsafe { kvm_set_tss_addr(&kvm_vm.vm, 0xf000_0000) }.context(error::SetVmParam)?;
unsafe { kvm_set_identity_map_addr(&kvm_vm.vm, &0xf000_3000) }
.context(error::SetVmParam)?;
Ok(())
}
}
#[cfg(test)]
mod test {
use crate::hv::kvm::{Kvm, KvmConfig};
use crate::hv::Hypervisor;
#[test]
fn test_get_supported_cpuid() {
let kvm = Kvm::new(KvmConfig::default()).unwrap();
let mut kvm_cpuid_exist = false;
let supported_cpuids = kvm.get_supported_cpuids().unwrap();
for cpuid in &supported_cpuids {
if cpuid.func == 0x4000_0000
&& cpuid.ebx.to_le_bytes() == *b"KVMK"
&& cpuid.ecx.to_le_bytes() == *b"VMKV"
&& cpuid.edx.to_le_bytes() == *b"M\0\0\0"
{
kvm_cpuid_exist = true;
}
}
assert!(kvm_cpuid_exist);
}
}