UDP has a weird edge-triggered EPOLLOUT behavior in Linux

[ppopth]

Describe the issue

As I mentioned in #3243 (comment), I found that only UDP has the weird edge-triggered epoll behavior in the output buffer like the one of the input buffer I mentioned in #2673 and fixed in #3243.

That is, if the file is already writable and someone writes more bytes to it, it will be writable again.

This thing happens only in UDP, but not other file types (TCP, Unix, Pipe, etc.). I don't know why, but it's better to file an issue here. I have some Linux tests for this issue as shown below.

To Reproduce

Note that, from the tests below, only UDP has trigger_writable set to be true.

diff --git a/src/test/epoll/test_epoll_edge.rs b/src/test/epoll/test_epoll_edge.rs
index 9163125fc..05e2ab99b 100644
--- a/src/test/epoll/test_epoll_edge.rs
+++ b/src/test/epoll/test_epoll_edge.rs
@@ -192,6 +192,56 @@ fn test_threads_multi_write(readfd: libc::c_int, writefd: libc::c_int) -> anyhow
     })
 }
 
+fn test_writable_when_write(
+    readfd: libc::c_int,
+    writefd: libc::c_int,
+    trigger_writable: bool,
+) -> anyhow::Result<()> {
+    let epollfd = epoll::epoll_create()?;
+
+    test_utils::run_and_close_fds(&[epollfd, readfd, writefd], || {
+        let mut event = epoll::EpollEvent::new(EpollFlags::EPOLLET | EpollFlags::EPOLLOUT, 0);
+        epoll::epoll_ctl(
+            epollfd,
+            epoll::EpollOp::EpollCtlAdd,
+            writefd,
+            Some(&mut event),
+        )?;
+
+        let timeout = Duration::from_millis(100);
+
+        let thread = std::thread::spawn(move || {
+            vec![
+                do_epoll_wait(epollfd, timeout, /* do_read= */ false),
+                do_epoll_wait(epollfd, timeout, /* do_read= */ false),
+            ]
+        });
+
+        // Wait for the waiter to block.
+        std::thread::sleep(timeout / 2);
+
+        // Write something to the write-end.
+        unistd::write(writefd, &[0])?;
+
+        let results = thread.join().unwrap();
+
+        ensure_ord!(results[0].epoll_res, ==, Ok(1));
+        ensure_ord!(results[0].duration, <, timeout);
+        ensure_ord!(results[0].events[0], ==, epoll::EpollEvent::new(EpollFlags::EPOLLOUT, 0));
+
+        if trigger_writable {
+            ensure_ord!(results[1].epoll_res, ==, Ok(1));
+            ensure_ord!(results[1].duration, <, timeout);
+            ensure_ord!(results[1].events[0], ==, epoll::EpollEvent::new(EpollFlags::EPOLLOUT, 0));
+        } else {
+            ensure_ord!(results[1].epoll_res, ==, Ok(0));
+            ensure_ord!(results[1].duration, >=, timeout);
+        }
+
+        Ok(())
+    })
+}
+
 fn test_oneshot_multi_write(readfd: libc::c_int, writefd: libc::c_int) -> anyhow::Result<()> {
     let epollfd = epoll::epoll_create()?;
 
@@ -384,7 +434,10 @@ fn main() -> anyhow::Result<()> {
     let mut tests: Vec<test_utils::ShadowTest<(), anyhow::Error>> = vec![];
 
     let mut add_tests =
-        |name: &str, swappable: bool, fds_init_helper: fn() -> (libc::c_int, libc::c_int)| {
+        |name: &str,
+         swappable: bool,
+         trigger_writable: bool,
+         fds_init_helper: fn() -> (libc::c_int, libc::c_int)| {
             // add details to the test names to avoid duplicates
             let append_args = |s, swapped: bool| {
                 if swappable {
@@ -412,6 +465,15 @@ fn main() -> anyhow::Result<()> {
                         extend_test(test_multi_write),
                         all_envs.clone(),
                     ),
+                    ShadowTest::new(
+                        &append_args("writable-when-write", swapped),
+                        move || {
+                            let (fd1, fd2) = fds_init_helper();
+                            let (readfd, writefd) = if swapped { (fd2, fd1) } else { (fd1, fd2) };
+                            test_writable_when_write(readfd, writefd, trigger_writable)
+                        },
+                        set![TestEnvironment::Libc],
+                    ),
                     ShadowTest::new(
                         &append_args("oneshot-multi-write", swapped),
                         extend_test(test_oneshot_multi_write),
@@ -431,42 +493,64 @@ fn main() -> anyhow::Result<()> {
             }
         };
 
-    add_tests("tcp", /* swappable = */ true, tcp_fds_init_helper);
-    add_tests("udp", /* swappable = */ false, udp_fds_init_helper);
+    add_tests(
+        "tcp",
+        /* swappable = */ true,
+        /* trigger_writable = */ false,
+        tcp_fds_init_helper,
+    );
+    add_tests(
+        "udp",
+        /* swappable = */ false,
+        /* trigger_writable = */ true,
+        udp_fds_init_helper,
+    );
     add_tests(
         "unix-stream",
         /* swappable = */ true,
+        /* trigger_writable = */ false,
         unix_stream_fds_init_helper,
     );
     add_tests(
         "unix-dgram",
         /* swappable = */ false,
+        /* trigger_writable = */ false,
         unix_dgram_fds_init_helper,
     );
     add_tests(
         "unix-seqpacket",
         /* swappable = */ true,
+        /* trigger_writable = */ false,
         unix_seqpacket_fds_init_helper,
     );
     add_tests(
         "unix-pair-stream",
         /* swappable = */ true,
+        /* trigger_writable = */ false,
         unix_pair_stream_fds_init_helper,
     );
     add_tests(
         "unix-pair-dgram",
         /* swappable = */ false,
+        /* trigger_writable = */ false,
         unix_pair_dgram_fds_init_helper,
     );
     add_tests(
         "unix-pair-seqpacket",
         /* swappable = */ true,
+        /* trigger_writable = */ false,
         unix_pair_seqpacket_fds_init_helper,
     );
-    add_tests("pipe", /* swappable = */ false, pipe_fds_init_helper);
+    add_tests(
+        "pipe",
+        /* swappable = */ false,
+        /* trigger_writable = */ false,
+        pipe_fds_init_helper,
+    );
     add_tests(
         "pipe-direct",
         /* swappable = */ false,
+        /* trigger_writable = */ false,
         pipe_direct_fds_init_helper,
     );

Operating System (please complete the following information):

• OS and version: Ubuntu 22.04.3 LTS
• Kernel version: Linux thinkpad-t14 6.5.0-15-generic # 15~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Fri Jan 12 18:54:30 UTC 2 x86_64 x86_64 x86_64 GNU/Linux

Shadow (please complete the following information):

• Version and build information: commit 0ec5536
• Which processes you are trying to run inside the Shadow simulation: the tests

Additional context