From 95b9d5f9ddde12ac683eaa7be117cc8b3b9667aa Mon Sep 17 00:00:00 2001
From: Changyuan Lyu <changyuanl@google.com>
Date: Mon, 16 Sep 2024 09:51:33 -0700
Subject: [PATCH] fix(virtio): monitor the waker with IORING_OP_POLL_ADD

With `IORING_POLL_ADD_MULTI`, we do not need to re-add the waker
to the ring.

Commit 0c4fc4e3ba8e also used IORING_OP_POLL_ADD for ioeventfds, which
was problematic because POLL_ADD does not clear the internal counter
of an eventfd when it creates a cqe. So once an ioeventfd is re-added
to the ring, it immediately generates a new cqe, despite that there is
no new MMIO yet. This causes the event loop thread to spin.

Signed-off-by: Changyuan Lyu <changyuanl@google.com>
---
 alioth/src/virtio/dev/dev.rs         | 4 ----
 alioth/src/virtio/worker/io_uring.rs | 8 ++------
 alioth/src/virtio/worker/mio.rs      | 4 ----
 3 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/alioth/src/virtio/dev/dev.rs b/alioth/src/virtio/dev/dev.rs
index d5276a9..528cff6 100644
--- a/alioth/src/virtio/dev/dev.rs
+++ b/alioth/src/virtio/dev/dev.rs
@@ -233,7 +233,6 @@ pub trait Backend<D: Virtio>: Send + 'static {
     type Data<'m>;
 
     fn register_waker(&mut self, token: u64) -> Result<Arc<Waker>>;
-    fn reregister_waker(&mut self, data: &mut Self::Data<'_>) -> Result<()>;
     fn activate_dev(
         &mut self,
         dev: &mut D,
@@ -310,9 +309,6 @@ where
                 }
             }
         }
-        if self.state == WorkerState::Running {
-            backend.reregister_waker(data)?;
-        }
         Ok(())
     }
 
diff --git a/alioth/src/virtio/worker/io_uring.rs b/alioth/src/virtio/worker/io_uring.rs
index c5fc757..1a149d5 100644
--- a/alioth/src/virtio/worker/io_uring.rs
+++ b/alioth/src/virtio/worker/io_uring.rs
@@ -85,8 +85,8 @@ where
 
     fn submit_waker(&mut self, data: &mut RingData) -> Result<()> {
         let fd = types::Fd(self.waker.0.as_raw_fd());
-        let read = opcode::Read::new(fd, self.efd_buffer.get() as *mut u8, 8);
-        let entry = read.build().user_data(self.waker_token);
+        let poll = opcode::PollAdd::new(fd, libc::EPOLLIN as _).multi(true);
+        let entry = poll.build().user_data(self.waker_token);
         unsafe { data.ring.submission().push(&entry) }.unwrap();
         Ok(())
     }
@@ -201,10 +201,6 @@ where
         Ok(self.waker.clone())
     }
 
-    fn reregister_waker(&mut self, data: &mut RingData<'_>) -> Result<()> {
-        self.submit_waker(data)
-    }
-
     fn activate_dev(
         &mut self,
         dev: &mut D,
diff --git a/alioth/src/virtio/worker/mio.rs b/alioth/src/virtio/worker/mio.rs
index 0674e81..9212793 100644
--- a/alioth/src/virtio/worker/mio.rs
+++ b/alioth/src/virtio/worker/mio.rs
@@ -137,10 +137,6 @@ where
         }
     }
 
-    fn reregister_waker(&mut self, _data: &mut ()) -> Result<()> {
-        Ok(())
-    }
-
     fn activate_dev(
         &mut self,
         dev: &mut D,