crosvm: add suspended flag to crosvm run

Add --suspended flag to crosvm run to start up a suspended VM.
For the VM to resume, need to run "crosvm resume --full $VM_SOCKET"

BUG=b:285933044
TEST=presubmit
TEST=start a VM with --suspended flag. Then, send resume command and
validate the VM is working.

Change-Id: I1674e4098491555bd4ccd16f84fe4109619469c1
Reviewed-on: https://chromium-review.googlesource.com/c/crosvm/crosvm/+/4615561
Reviewed-by: Steven Moreland <smoreland@google.com>
Reviewed-by: Noah Gold <nkgold@google.com>
Commit-Queue: Elie Kheirallah <khei@google.com>
Reviewed-by: Frederick Mayle <fmayle@google.com>

devices/src/lib.rs

@@ -388,8 +388,6 @@ async fn restore_handler(
     }
 
     {
-        let _sleep_guard = SleepGuard::new(buses)?;
-
         guest_memory.restore(snapshot_root.guest_memory_metadata, &mut mem_file)?;
 
         for bus in buses {
@@ -463,6 +461,10 @@ async fn handle_command_tube(
                             .context("Failed to send response")?;
                     }
                     DeviceControlCommand::RestoreDevices { restore_path: path } => {
+                        assert!(
+                            _sleep_guard.is_some(),
+                            "devices must be sleeping to restore"
+                        );
                         if let Err(e) =
                             restore_handler(path.as_path(), &guest_memory, &[&*io_bus, &*mmio_bus])
                                 .await

src/crosvm/cmdline.rs

@@ -2036,6 +2036,12 @@ pub struct RunCommand {
     ///     revision=NUM - revision
     pub stub_pci_device: Vec<StubPciParameters>,
 
+    #[argh(switch)]
+    #[serde(skip)] // TODO(b/255223604)
+    #[merge(strategy = overwrite_option)]
+    /// start a VM with vCPUs and devices suspended
+    pub suspended: Option<bool>,
+
     #[argh(option, long = "swap", arg_name = "PATH")]
     #[serde(skip)] // TODO(b/255223604)
     #[merge(strategy = overwrite_option)]
@@ -2706,6 +2712,7 @@ impl TryFrom<RunCommand> for super::config::Config {
 
         cfg.swap_dir = cmd.swap_dir;
         cfg.restore_path = cmd.restore;
+        cfg.suspended = cmd.suspended.unwrap_or_default();
 
         if let Some(mut socket_path) = cmd.socket {
             if socket_path.is_dir() {

src/crosvm/config.rs

@@ -1200,6 +1200,7 @@ pub struct Config {
     pub sound: Option<PathBuf>,
     pub strict_balloon: bool,
     pub stub_pci_devices: Vec<StubPciParameters>,
+    pub suspended: bool,
     pub swap_dir: Option<PathBuf>,
     pub swiotlb: Option<u64>,
     #[cfg(target_os = "android")]
@@ -1409,7 +1410,6 @@ impl Default for Config {
             slirp_capture_file: None,
             #[cfg(all(windows, feature = "audio"))]
             snd_split_config: None,
-            swap_dir: None,
             socket_path: None,
             #[cfg(feature = "tpm")]
             software_tpm: false,
@@ -1417,6 +1417,8 @@ impl Default for Config {
             sound: None,
             strict_balloon: false,
             stub_pci_devices: Vec::new(),
+            suspended: false,
+            swap_dir: None,
             swiotlb: None,
             #[cfg(target_os = "android")]
             task_profiles: Vec::new(),

src/crosvm/sys/unix.rs

@@ -2784,7 +2784,22 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
     android::set_process_profiles(&cfg.task_profiles)?;
 
     #[allow(unused_mut)]
-    let mut run_mode = VmRunMode::Running;
+    let mut run_mode = if cfg.suspended {
+        // Sleep devices before creating vcpus.
+        device_ctrl_tube
+            .send(&DeviceControlCommand::SleepDevices)
+            .context("send command to devices control socket")?;
+        match device_ctrl_tube
+            .recv()
+            .context("receive from devices control socket")?
+        {
+            VmResponse::Ok => (),
+            resp => bail!("device sleep failed: {}", resp),
+        }
+        VmRunMode::Suspending
+    } else {
+        VmRunMode::Running
+    };
     #[cfg(feature = "gdb")]
     if to_gdb_channel.is_some() {
         // Wait until a GDB client attaches

src/sys/windows.rs

@@ -190,6 +190,7 @@ use vm_control::VcpuControl;
 use vm_control::VmMemoryRegionState;
 use vm_control::VmMemoryRequest;
 use vm_control::VmRequest;
+use vm_control::VmResponse;
 use vm_control::VmRunMode;
 use vm_memory::GuestAddress;
 use vm_memory::GuestMemory;
@@ -1078,6 +1079,7 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
     restore_path: Option<PathBuf>,
     control_server_path: Option<PathBuf>,
     force_s2idle: bool,
+    suspended: bool,
 ) -> Result<ExitState> {
     let (ipc_main_loop_tube, proto_main_loop_tube, _service_ipc) =
         start_service_ipc_listener(service_pipe_name)?;
@@ -1171,6 +1173,24 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
     let vcpu_boxes: Arc<Mutex<Vec<Box<dyn VcpuArch>>>> = Arc::new(Mutex::new(Vec::new()));
     let run_mode_arc = Arc::new(VcpuRunMode::default());
 
+    let run_mode_state = if suspended {
+        // Sleep devices before creating vcpus.
+        device_ctrl_tube
+            .send(&DeviceControlCommand::SleepDevices)
+            .context("send command to devices control socket")?;
+        match device_ctrl_tube
+            .recv()
+            .context("receive from devices control socket")?
+        {
+            VmResponse::Ok => (),
+            resp => bail!("device sleep failed: {}", resp),
+        }
+        run_mode_arc.set_and_notify(VmRunMode::Suspending);
+        VmRunMode::Suspending
+    } else {
+        VmRunMode::Running
+    };
+
     // If we are restoring from a snapshot, then start suspended.
     if restore_path.is_some() {
         run_mode_arc.set_and_notify(VmRunMode::Suspending);
@@ -1236,7 +1256,7 @@ fn run_control<V: VmArch + 'static, Vcpu: VcpuArch + 'static>(
             // Other platforms (unix) have multiple modes they could start in (e.g. starting for
             // guest kernel debugging, etc). If/when we support those modes on Windows, we'll need
             // to enter that mode here rather than VmRunMode::Running.
-            VcpuControl::RunState(VmRunMode::Running),
+            VcpuControl::RunState(run_mode_state),
         );
     }
 
@@ -2432,6 +2452,7 @@ where
         cfg.restore_path,
         cfg.socket_path,
         cfg.force_s2idle,
+        cfg.suspended,
     )
 }
 

vm_control/src/lib.rs

@@ -2183,8 +2183,8 @@ pub fn do_restore(
     vcpu_size: usize,
     mut restore_irqchip: impl FnMut(serde_json::Value) -> anyhow::Result<()>,
 ) -> anyhow::Result<()> {
-    let _guard = VcpuSuspendGuard::new(&kick_vcpus, vcpu_size)?;
-    let _device_guard = DeviceSleepGuard::new(device_control_tube)?;
+    let _guard = VcpuSuspendGuard::new(&kick_vcpus, vcpu_size);
+    let _devices_guard = DeviceSleepGuard::new(device_control_tube)?;
 
     // Restore IrqChip
     let irq_path = restore_path.with_extension("irqchip");