Interrupt enclu execution on main enclave thread exiting

intel-sgx/enclave-runner/src/tcs.rs

@@ -14,13 +14,21 @@ use std::os::raw::c_void;
 
 use sgx_isa::Enclu;
 use sgxs::loader::Tcs;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+use std::sync::atomic::Ordering;
+
+thread_local! {
+    pub static EXITING: RefCell<Option<Arc<AtomicBool>>> = RefCell::new(None);
+}
 
 pub(crate) type DebugBuffer = [u8; 1024];
 
 #[derive(Debug)]
 pub enum CoResult<Y, R> {
     Yield(Y),
     Return(R),
+    Abort,
 }
 
 #[derive(Debug)]
@@ -39,58 +47,61 @@ impl<T: Tcs> Usercall<T> {
     pub fn coreturn(
         self,
         retval: (u64, u64),
-        debug_buf: Option<&RefCell<DebugBuffer>>,
+        exiting: &Arc<AtomicBool>,
+        debug_buf: Option<&RefCell<DebugBuffer>>
     ) -> ThreadResult<T> {
-        coenter(self.tcs, 0, retval.0, retval.1, 0, 0, debug_buf)
+        coenter(self.tcs, 0, retval.0, retval.1, 0, 0, exiting, debug_buf)
     }
 
     pub fn tcs_address(&self) -> *mut c_void {
         self.tcs.address()
     }
 }
 
-pub(crate) fn coenter<T: Tcs>(
-    tcs: T,
-    mut p1: u64,
-    mut p2: u64,
-    mut p3: u64,
-    mut p4: u64,
-    mut p5: u64,
-    debug_buf: Option<&RefCell<DebugBuffer>>,
-) -> ThreadResult<T> {
-    /// Check if __vdso_sgx_enter_enclave exists. We're using weak linkage, so
-    /// it might not.
-    #[cfg(target_os = "linux")]
-    fn has_vdso_sgx_enter_enclave() -> bool {
-        unsafe {
-            let addr: usize;
-            asm!("
+/// Check if __vdso_sgx_enter_enclave exists. We're using weak linkage, so
+/// it might not.
+#[cfg(target_os = "linux")]
+pub(crate) fn has_vdso_sgx_enter_enclave() -> bool {
+    unsafe {
+        let addr: usize;
+        asm!("
 .weak __vdso_sgx_enter_enclave
 .type __vdso_sgx_enter_enclave, function
-                mov __vdso_sgx_enter_enclave@GOTPCREL(%rip), {}
-                jmp 1f
-
-                // Strongly link to another symbol in the VDSO, so that the
-                // linker will include a DT_NEEDED entry for `linux-vdso.so.1`.
-                // This doesn't happen automatically because rustc passes
-                // `--as-needed` to the linker. This is never executed because
-                // of the unconditional jump above.
+            mov __vdso_sgx_enter_enclave@GOTPCREL(%rip), {}
+            jmp 1f
+
+            // Strongly link to another symbol in the VDSO, so that the
+            // linker will include a DT_NEEDED entry for `linux-vdso.so.1`.
+            // This doesn't happen automatically because rustc passes
+            // `--as-needed` to the linker. This is never executed because
+            // of the unconditional jump above.
 .global __vdso_clock_gettime
 .type __vdso_clock_gettime, function
-                call __vdso_clock_gettime@PLT
+            call __vdso_clock_gettime@PLT
 
 1:
-                ", out(reg) addr, options(nomem, nostack, att_syntax));
-            addr != 0
-        }
+            ", out(reg) addr, options(nomem, nostack, att_syntax));
+        addr != 0
     }
+}
 
-    #[cfg(not(target_os = "linux"))]
-    fn has_vdso_sgx_enter_enclave() -> bool {
-        false
-    }
+#[cfg(not(target_os = "linux"))]
+pub(crate) fn has_vdso_sgx_enter_enclave() -> bool {
+    false
+}
 
-    let sgx_result: u32;
+pub(crate) fn coenter<T: Tcs>(
+    tcs: T,
+    mut p1: u64,
+    mut p2: u64,
+    mut p3: u64,
+    mut p4: u64,
+    mut p5: u64,
+    exiting: &Arc<AtomicBool>,
+    debug_buf: Option<&RefCell<DebugBuffer>>
+) -> ThreadResult<T> {
+
+    let mut enclu_leaf = Enclu::EEnter as u32;
 
     unsafe {
         let mut uninit_debug_buf: std::mem::MaybeUninit<DebugBuffer>;
@@ -103,6 +114,7 @@ pub(crate) fn coenter<T: Tcs>(
             }
         };
         if has_vdso_sgx_enter_enclave() {
+
             #[repr(C)]
             #[derive(Default)]
             struct SgxEnclaveRun {
@@ -137,6 +149,9 @@ pub(crate) fn coenter<T: Tcs>(
                 ..Default::default()
             };
             let ret: i32;
+
+            // set exiting flag in thread local storage variable to check it in signal handler in case of a signal
+            EXITING.with(|cell| *cell.borrow_mut() = Some(exiting.clone()));
             asm!("
                     sub $0x8, %rsp                   // align stack
                     push {}                          // push argument: run
@@ -162,11 +177,14 @@ pub(crate) fn coenter<T: Tcs>(
                 lateout("r15") _, // V
                 options(att_syntax)
             );
+            // Enclave work is done - unset TLS variable
+            EXITING.with(|cell| { *cell.borrow_mut() = None;});
+
             if ret == 0 {
-                sgx_result = run.function;
-                match sgx_result.try_into() {
-                    Ok(Enclu::EExit) => { /* normal case */ },
-                    Ok(Enclu::EResume) => {
+                enclu_leaf = run.function;
+                match enclu_leaf.try_into() {
+                    Ok(Enclu::EExit) => { /* normal case or enclave work was intervened by the sighup */ },
+                    Ok(Enclu::EResume) => { /* given vdso code this never may be the case */
                         if let Some(mut debug_buf) = debug_buf {
                             let _ = write!(&mut debug_buf[..], "Enclave triggered exception: {:?}\0", run);
                         } else {
@@ -183,31 +201,42 @@ pub(crate) fn coenter<T: Tcs>(
                 panic!("Error entering enclave (VDSO): ret = {:?}, run = {:?}", std::io::Error::from_raw_os_error(-ret), run);
             }
         } else {
-            asm!("
-                    lea 1f(%rip), %rcx // set SGX AEP
-                    xchg {0}, %rbx
-1:                  enclu
-                    xchg %rbx, {0}
-                ",
-                inout(reg) tcs.address() => _, // rbx is used internally by LLVM and cannot be used as an operand for inline asm (#84658)
-                inout("eax") Enclu::EEnter as u32 => sgx_result,
-                out("rcx") _,
-                inout("rdx") p3,
-                inout("rdi") p1,
-                inout("rsi") p2,
-                inout("r8") p4,
-                inout("r9") p5,
-                inout("r10") debug_buf_ptr => _,
-                out("r11") _,
-                options(nostack, att_syntax)
-            );
+            while !exiting.load(Ordering::SeqCst) && enclu_leaf != (Enclu::EExit as u32) {
+                asm!("
+                        lea 1f(%rip), %rcx // set SGX AEP
+                        push %rbx          // rbx is used internally by LLVM and cannot be designated as being clobbered. Store/restoring it
+                        mov {0}, %rbx
+                        enclu
+1:                      pop %rbx
+                    ",
+                    inout(reg) tcs.address() => _, // rbx is used internally by LLVM and cannot be used as an operand for inline asm (#84658)
+                    inout("eax") enclu_leaf as u32 => enclu_leaf,
+                    out("rcx") _,
+                    inout("rdx") p3,
+                    inout("rdi") p1,
+                    inout("rsi") p2,
+                    inout("r8") p4,
+                    inout("r9") p5,
+                    inout("r10") debug_buf_ptr => _,
+                    out("r11") _,
+                    lateout("r12") _, // these may be clobbered in case of AEX
+                    lateout("r13") _, // V
+                    lateout("r14") _, // V
+                    lateout("r15") _, // V
+                    options(nostack, att_syntax)
+                );
+            }
         }
     };
 
-    if sgx_result != (Enclu::EExit as u32) {
-        panic!("Invalid return value in EAX! eax={}", sgx_result);
+    if exiting.load(Ordering::SeqCst) {
+        return CoResult::Abort
     }
 
+    if enclu_leaf != (Enclu::EExit as u32) {
+        panic!("Invalid return value in EAX! eax={}", enclu_leaf);
+    }
+
     if p1 == 0 {
         CoResult::Return((tcs, p2, p3))
     } else {

intel-sgx/enclave-runner/src/usercalls/mod.rs

@@ -7,13 +7,15 @@
 use std::alloc::{GlobalAlloc, Layout, System};
 use std::cell::RefCell;
 use std::collections::VecDeque;
-use std::io::{self, ErrorKind as IoErrorKind, Read, Result as IoResult};
+use std::io::{self, stderr, Write, ErrorKind as IoErrorKind, Read, Result as IoResult};
 use std::pin::Pin;
 use std::result::Result as StdResult;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::sync::Arc;
 use std::task::{Context, Poll, Waker};
 use std::thread::{self, JoinHandle};
+#[cfg(unix)]
+use std::os::unix::thread::JoinHandleExt;
 use std::time::{self, Duration};
 use std::{cmp, fmt, str};
 
@@ -38,9 +40,10 @@ use ipc_queue::{self, DescriptorGuard, Identified, QueueEvent};
 use sgxs::loader::Tcs as SgxsTcs;
 
 use crate::loader::{EnclavePanic, ErasedTcs};
-use crate::tcs::{self, CoResult, ThreadResult};
+use crate::tcs::{self, CoResult, ThreadResult, EXITING};
 use self::abi::dispatch;
 use self::interface::{Handler, OutputBuffer};
+use sgx_isa::Enclu;
 
 pub(crate) mod abi;
 mod interface;
@@ -647,7 +650,7 @@ pub(crate) struct EnclaveState {
     event_queues: FnvHashMap<TcsAddress, PendingEvents>,
     fds: Mutex<FnvHashMap<Fd, Arc<AsyncFileDesc>>>,
     last_fd: AtomicUsize,
-    exiting: AtomicBool,
+    exiting: Arc<AtomicBool>,
     usercall_ext: Box<dyn UsercallExtension>,
     threads_queue: crossbeam::queue::SegQueue<StoppedTcs>,
     forward_panics: bool,
@@ -667,15 +670,15 @@ enum CoEntry {
 }
 
 impl Work {
-    fn do_work(self, io_send_queue: &mut tokio::sync::mpsc::UnboundedSender<UsercallSendData>) {
+    fn do_work(self, io_send_queue: &mut tokio::sync::mpsc::UnboundedSender<UsercallSendData>, exiting: &Arc<AtomicBool>) {
         let buf = RefCell::new([0u8; 1024]);
         let usercall_send_data = match self.entry {
             CoEntry::Initial(erased_tcs, p1, p2, p3, p4, p5) => {
-                let coresult = tcs::coenter(erased_tcs, p1, p2, p3, p4, p5, Some(&buf));
+                let coresult = tcs::coenter(erased_tcs, p1, p2, p3, p4, p5, &exiting, Some(&buf));
                 UsercallSendData::Sync(coresult, self.tcs, buf)
             }
             CoEntry::Resume(usercall, coresult) => {
-                let coresult = usercall.coreturn(coresult, Some(&buf));
+                let coresult = usercall.coreturn(coresult, &exiting, Some(&buf));
                 UsercallSendData::Sync(coresult, self.tcs, buf)
             }
         };
@@ -739,7 +742,7 @@ impl EnclaveState {
             event_queues,
             fds: Mutex::new(fds),
             last_fd,
-            exiting: AtomicBool::new(false),
+            exiting: Arc::new(AtomicBool::new(false)),
             usercall_ext,
             threads_queue,
             forward_panics,
@@ -935,6 +938,9 @@ impl EnclaveState {
                         };
                         tokio::task::spawn_local(fut);
                     }
+                    UsercallSendData::Sync(CoResult::Abort, _, _) => {
+                        // enclave interuption - do nothing
+                    }
                 };
             }
             unreachable!();
@@ -964,21 +970,28 @@ impl EnclaveState {
             num_of_worker_threads: usize,
             work_receiver: crossbeam::crossbeam_channel::Receiver<Work>,
             io_queue_send: tokio::sync::mpsc::UnboundedSender<UsercallSendData>,
+            exiting: Arc<AtomicBool>
         ) -> Vec<JoinHandle<()>> {
             let mut thread_handles = vec![];
             for _ in 0..num_of_worker_threads {
                 let work_receiver = work_receiver.clone();
                 let mut io_queue_send = io_queue_send.clone();
+                let exiting = exiting.clone();
 
                 thread_handles.push(thread::spawn(move || {
                     while let Ok(work) = work_receiver.recv() {
-                        work.do_work(&mut io_queue_send);
+                        work.do_work(&mut io_queue_send, &exiting);
                     }
                 }));
             }
             thread_handles
         }
 
+        #[cfg(unix)]
+        if tcs::has_vdso_sgx_enter_enclave() {
+            catch_sighup()
+        };
+
         let (io_queue_send, io_queue_receive) = tokio::sync::mpsc::unbounded_channel();
 
         let (work_sender, work_receiver) = crossbeam::crossbeam_channel::unbounded();
@@ -987,12 +1000,20 @@ impl EnclaveState {
             .expect("Work sender couldn't send data to receiver");
 
         let join_handlers =
-            create_worker_threads(num_of_worker_threads, work_receiver, io_queue_send.clone());
+            create_worker_threads(num_of_worker_threads, work_receiver, io_queue_send.clone(), enclave.exiting.clone());
         // main syscall polling loop
         let main_result =
             EnclaveState::syscall_loop(enclave.clone(), io_queue_receive, io_queue_send, work_sender);
 
         for handler in join_handlers {
+            #[cfg(unix)]
+            if tcs::has_vdso_sgx_enter_enclave() {
+                // * The enclave thread may be in a long-running `AEX/enclu[ERESUME]` loop.
+                // for vdso case: execution control back to the enclave-runner worker thread returned
+                //    by means of issued signal and rewriting IP in signal handler.
+                unsafe { libc::pthread_kill(handler.as_pthread_t() as _, signal::SIGHUP as _); }
+                // for non-vdso case: execution control returned using AEP set next after Enclu to break AEX/enclu[EResume] loop (signal not required).
+            }
             let _ = handler.join();
         }
         return main_result;
@@ -1214,6 +1235,33 @@ extern "C" fn handle_trap(_signo: c_int, _info: *mut siginfo_t, context: *mut c_
     return;
 }
 
+#[cfg(unix)]
+fn catch_sighup() {
+    unsafe {
+        extern "C" fn handle_sighup(_signo: c_int, _info: *mut siginfo_t, context: *mut c_void) {
+            eprintln!("SIGHUP triggered, thread_id: {:?}", std::thread::current().id());
+            let instruction_ptr = unsafe { (*(context as *mut ucontext_t)).uc_mcontext.gregs[REG_RIP as usize] as *const u8};
+            // enclu instruction code
+            const ENCLU: [u8; 3] = [0x0f, 0x01, 0xd7];
+            let is_enclu = ENCLU.iter().enumerate().all(|(idx, v)| {
+                unsafe { *instruction_ptr.offset(idx as isize) == *v }
+            });
+            if is_enclu && EXITING.with(|cell| cell.borrow().as_ref().is_some_and(|val| val.load(Ordering::SeqCst) == true)) {
+                // Interrupt enclave execution by setting IP to the instruction following the ENCLU to mimic normal ENCLU[EEXIT])
+                unsafe {
+                    (*(context as *mut ucontext_t)).uc_mcontext.gregs[REG_RIP as usize] += ENCLU.len() as i64;
+                    (*(context as *mut ucontext_t)).uc_mcontext.gregs[REG_RAX as usize] = Enclu::EExit as i64;
+                }
+            }
+            let _ = stderr().flush();
+        }
+
+        let hdl = signal::SigHandler::SigAction(handle_sighup);
+        let sig_action = signal::SigAction::new(hdl, signal::SaFlags::empty(), signal::SigSet::empty());
+        signal::sigaction(signal::SIGHUP, &sig_action).unwrap();
+    }
+}
+
 #[cfg(unix)]
 /* Raising Sigtrap to allow debugger to take control.
  * Here, we also store tcs in rbx, so that the debugger could read it, to