acpipm: implement suspend and resume mechanism

For suspend request from VM, will write suspend event and notify
crosvm main process to pause VCPUs.

For resume request, it is not from VM itself but by the resume
command through crosvm socket. Resume request will notify the PM
device to fill its wakeup registers with wakeup event so that
when VCPUs start to run, VM can know there is wakeup from outside.

BUG=chromium:1018674
TEST=cargo test -p devices

Change-Id: I4724ffee10150065a62bf520076c16cbc70b7749
Signed-off-by: Chuanxiao Dong <chuanxiao.dong@intel.corp-partner.google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/crosvm/+/2035169
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Tomasz Jeznach <tjeznach@chromium.org>

devices/src/acpi.rs

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-use crate::BusDevice;
+use crate::{BusDevice, BusResumeDevice};
 use sys_util::{error, warn, EventFd};
 
 /// ACPI PM resource for handling OS suspend/resume request
@@ -41,6 +41,8 @@ const SLEEP_CONTROL: u16 = 6;
 const SLEEP_STATUS: u16 = 7;
 const BITMASK_PM1CNT_SLEEP_ENABLE: u16 = 0x2000;
 const BITMASK_SLEEPCNT_SLEEP_ENABLE: u8 = 0x20;
+const BITMASK_PM1CNT_WAKE_STATUS: u16 = 0x8000;
+const BITMASK_SLEEPCNT_WAKE_STATUS: u8 = 0x80;
 
 impl BusDevice for ACPIPMResource {
     fn debug_label(&self) -> String {
@@ -113,3 +115,13 @@ impl BusDevice for ACPIPMResource {
         };
     }
 }
+
+impl BusResumeDevice for ACPIPMResource {
+    fn resume_imminent(&mut self) {
+        let val = self.pm1_status;
+        self.pm1_status = val | BITMASK_PM1CNT_WAKE_STATUS;
+
+        let val = self.sleep_status;
+        self.sleep_status = val | BITMASK_SLEEPCNT_WAKE_STATUS;
+    }
+}

devices/src/bus.rs

@@ -37,6 +37,12 @@ pub trait BusDevice: Send {
     fn on_sandboxed(&mut self) {}
 }
 
+pub trait BusResumeDevice: Send {
+    /// notify the devices which are invoked
+    /// before the VM resumes form suspend.
+    fn resume_imminent(&mut self) {}
+}
+
 #[derive(Debug)]
 pub enum Error {
     /// The insertion failed because the new device overlapped with an old device.
@@ -104,9 +110,13 @@ impl PartialOrd for BusRange {
 ///
 /// This doesn't have any restrictions on what kind of device or address space this applies to. The
 /// only restriction is that no two devices can overlap in this address space.
+///
+/// the 'resume_notify_devices' contains the devices which requires to be notified before the system
+/// resume back from S3 suspended state.
 #[derive(Clone)]
 pub struct Bus {
     devices: BTreeMap<BusRange, Arc<Mutex<dyn BusDevice>>>,
+    resume_notify_devices: Vec<Arc<Mutex<dyn BusResumeDevice>>>,
 }
 
 impl Bus {
@@ -114,6 +124,7 @@ impl Bus {
     pub fn new() -> Bus {
         Bus {
             devices: BTreeMap::new(),
+            resume_notify_devices: Vec::new(),
         }
     }
 
@@ -208,6 +219,19 @@ impl Bus {
             false
         }
     }
+
+    /// Register `device` for notifications of VM resume from suspend.
+    pub fn notify_on_resume(&mut self, device: Arc<Mutex<dyn BusResumeDevice>>) {
+        self.resume_notify_devices.push(device);
+    }
+
+    /// Call `notify_resume` to notify the device that suspend resume is imminent.
+    pub fn notify_resume(&mut self) {
+        let devices = self.resume_notify_devices.clone();
+        for dev in devices {
+            dev.lock().resume_imminent();
+        }
+    }
 }
 
 #[cfg(test)]

devices/src/lib.rs

@@ -25,7 +25,7 @@ pub mod virtio;
 
 pub use self::acpi::ACPIPMResource;
 pub use self::bus::Error as BusError;
-pub use self::bus::{Bus, BusDevice, BusRange};
+pub use self::bus::{Bus, BusDevice, BusRange, BusResumeDevice};
 pub use self::cmos::Cmos;
 pub use self::i8042::I8042Device;
 pub use self::ioapic::Ioapic;

src/linux.rs

@@ -1638,6 +1638,7 @@ fn run_control(
     #[derive(PollToken)]
     enum Token {
         Exit,
+        Suspend,
         ChildSignal,
         CheckAvailableMemory,
         LowMemory,
@@ -1652,6 +1653,7 @@ fn run_control(
 
     let poll_ctx = PollContext::build_with(&[
         (&linux.exit_evt, Token::Exit),
+        (&linux.suspend_evt, Token::Suspend),
         (&sigchld_fd, Token::ChildSignal),
     ])
     .map_err(Error::PollContextAdd)?;
@@ -1744,6 +1746,14 @@ fn run_control(
                     info!("vcpu requested shutdown");
                     break 'poll;
                 }
+                Token::Suspend => {
+                    info!("VM requested suspend");
+                    linux.suspend_evt.read().unwrap();
+                    run_mode_arc.set_and_notify(VmRunMode::Suspending);
+                    for handle in &vcpu_handles {
+                        let _ = handle.kill(SIGRTMIN() + 0);
+                    }
+                }
                 Token::ChildSignal => {
                     // Print all available siginfo structs, then exit the loop.
                     while let Some(siginfo) = sigchld_fd.read().map_err(Error::SignalFd)? {
@@ -1879,6 +1889,17 @@ fn run_control(
                                             VmRunMode::Exiting => {
                                                 break 'poll;
                                             }
+                                            VmRunMode::Running => {
+                                                if let VmRunMode::Suspending =
+                                                    *run_mode_arc.mtx.lock()
+                                                {
+                                                    linux.io_bus.notify_resume();
+                                                }
+                                                run_mode_arc.set_and_notify(VmRunMode::Running);
+                                                for handle in &vcpu_handles {
+                                                    let _ = handle.kill(SIGRTMIN() + 0);
+                                                }
+                                            }
                                             other => {
                                                 run_mode_arc.set_and_notify(other);
                                                 for handle in &vcpu_handles {
@@ -1937,6 +1958,7 @@ fn run_control(
         for event in events.iter_hungup() {
             match event.token() {
                 Token::Exit => {}
+                Token::Suspend => {}
                 Token::ChildSignal => {}
                 Token::CheckAvailableMemory => {}
                 Token::LowMemory => {}

x86_64/src/lib.rs

@@ -734,12 +734,13 @@ impl X8664arch {
         let pm = Arc::new(Mutex::new(devices::ACPIPMResource::new(suspend_evt)));
         io_bus
             .insert(
-                pm,
+                pm.clone(),
                 devices::acpi::ACPIPM_RESOURCE_BASE,
                 devices::acpi::ACPIPM_RESOURCE_LEN,
                 false,
             )
             .unwrap();
+        io_bus.notify_on_resume(pm);
 
         Ok(io_bus)
     }