Skip to content

Commit

Permalink
Check thread local exiting flag in sighup handler
Browse files Browse the repository at this point in the history
  • Loading branch information
Max K committed Jul 17, 2023
1 parent 6d58e04 commit 3b3ad1a
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 98 deletions.
126 changes: 38 additions & 88 deletions intel-sgx/enclave-runner/src/tcs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@ use sgxs::loader::Tcs;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::sync::atomic::Ordering;
use libc::{c_int, c_long};

thread_local! {
pub static EXITING: RefCell<Option<Arc<AtomicBool>>> = RefCell::new(None);
}

pub(crate) type DebugBuffer = [u8; 1024];

Expand Down Expand Up @@ -100,22 +103,6 @@ struct SgxEnclaveRun {
reserved: [u64; 27],
}

const RET_BY_SIGNAL: c_int = -10;

#[no_mangle]
extern "C" fn sgx_user_handler(_rdi: c_long, _rsi: c_long, _rdx: c_long,
_rsp: c_long, _r8: c_long, _r9: c_long,
run: *const SgxEnclaveRun) -> c_int {
let run: &SgxEnclaveRun = unsafe { &*run };
if _rdi == 0 && _rsi == 0 && _rdx == 0 {
// this is AEX caused by sighup signal handler
// return RET_BY_SIGNAL to enclave calling side
RET_BY_SIGNAL
} else {
0
}
}

pub(crate) fn coenter<T: Tcs>(
tcs: T,
mut p1: u64,
Expand All @@ -140,37 +127,6 @@ pub(crate) fn coenter<T: Tcs>(
}
};
if has_vdso_sgx_enter_enclave() {
let asm_sgx_user_handler: usize;
asm!("
jmp 1f
0:
// Inline function to call rust-based user_handler
push %rdi // Store save-by-caller registers so these won't get clobbered by handler
push %rsi
push %rdx
push %rcx
push %r8
push %r9
push %r10
// Call rust-based sgx_user_handler
movq 8*8(%rsp), %r10 # store 7th arg of sgx_user_handler on stack
push %r10
push %r10 # ensure the stack is still aligned
callq sgx_user_handler@PLT
popq %r10 # undo alignment
popq %r10
popq %r10 # restore save-by-caller registers
popq %r9
popq %r8
popq %rcx
popq %rdx
popq %rsi
popq %rdi
retq
1:
lea 0b(%rip), {}
", out(reg) asm_sgx_user_handler, options(att_syntax)
);

impl fmt::Debug for SgxEnclaveRun {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
Expand All @@ -190,51 +146,45 @@ pub(crate) fn coenter<T: Tcs>(

let mut run = SgxEnclaveRun {
tcs: tcs.address() as _,
user_handler: asm_sgx_user_handler as _,
..Default::default()
};

let mut ret: i32;

loop {
asm!("
sub $0x8, %rsp // align stack
push {} // push argument: run
.weak __vdso_sgx_enter_enclave
.type __vdso_sgx_enter_enclave, function
call __vdso_sgx_enter_enclave@PLT
add $0x10, %rsp // restore stack pointer
",
in(reg) &mut run,
lateout("eax") ret,
/* rbx unused */
inout("rcx") enclu_leaf => _,
inout("rdx") p3,
inout("rdi") p1,
inout("rsi") p2,
inout("r8") p4,
inout("r9") p5,
inout("r10") debug_buf_ptr => _,
lateout("r11") _,
lateout("r12") _, // these may be clobbered in case of AEX
lateout("r13") _, // V
lateout("r14") _, // V
lateout("r15") _, // V
options(att_syntax)
);
// set exiting flag in thread local storage varible to check it in signal handler in case of a signal
EXITING.with(|cell| *cell.borrow_mut() = Some(exiting.clone()));
asm!("
sub $0x8, %rsp // align stack
push {} // push argument: run
.weak __vdso_sgx_enter_enclave
.type __vdso_sgx_enter_enclave, function
call __vdso_sgx_enter_enclave@PLT
add $0x10, %rsp // restore stack pointer
",
in(reg) &mut run,
lateout("eax") ret,
/* rbx unused */
inout("rcx") Enclu::EEnter as u64 => _,
inout("rdx") p3,
inout("rdi") p1,
inout("rsi") p2,
inout("r8") p4,
inout("r9") p5,
inout("r10") debug_buf_ptr => _,
lateout("r11") _,
lateout("r12") _, // these may be clobbered in case of AEX
lateout("r13") _, // V
lateout("r14") _, // V
lateout("r15") _, // V
options(att_syntax)
);
// Enclave work is done - unset TLS variable
EXITING.with(|cell| { *cell.borrow_mut() = None;});

if !exiting.load(Ordering::SeqCst) && ret == RET_BY_SIGNAL {
// Enclu interrupted by a sighup handler before exiting - resume enclu
enclu_leaf = Enclu::EResume as _;
} else {
break;
}
}
if ret == 0 || ret == RET_BY_SIGNAL {
if ret == 0 {
enclu_leaf = run.function;
match enclu_leaf.try_into() {
Ok(Enclu::EExit) => { /* normal case */ },
Ok(Enclu::EResume) => {
Ok(Enclu::EExit) => { /* normal case or enclave work was intervened by the sighup */ },
Ok(Enclu::EResume) => { /* given vdso code this never may be the case */
if let Some(mut debug_buf) = debug_buf {
let _ = write!(&mut debug_buf[..], "Enclave triggered exception: {:?}\0", run);
} else {
Expand All @@ -254,10 +204,10 @@ pub(crate) fn coenter<T: Tcs>(
while !exiting.load(Ordering::SeqCst) && enclu_leaf != (Enclu::EExit as u32) {
asm!("
lea 1f(%rip), %rcx // set SGX AEP
push %rbx // store original rbx value on the stack
push %rbx
mov {0}, %rbx
enclu
1: pop %rbx // restore original rbx value in case we interrupt enclave during AEX
1: pop %rbx
",
inout(reg) tcs.address() => _, // rbx is used internally by LLVM and cannot be used as an operand for inline asm (#84658)
inout("eax") enclu_leaf as u32 => enclu_leaf,
Expand Down
27 changes: 17 additions & 10 deletions intel-sgx/enclave-runner/src/usercalls/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,11 @@ use ipc_queue::{self, DescriptorGuard, Identified, QueueEvent};
use sgxs::loader::Tcs as SgxsTcs;

use crate::loader::{EnclavePanic, ErasedTcs};
use crate::tcs::{self, CoResult, ThreadResult};
use crate::tcs::{self, CoResult, ThreadResult, EXITING};
use self::abi::dispatch;
use self::interface::{Handler, OutputBuffer};
use crate::tcs::has_vdso_sgx_enter_enclave;
use sgx_isa::Enclu;

pub(crate) mod abi;
mod interface;
Expand Down Expand Up @@ -990,7 +991,10 @@ impl EnclaveState {
thread_handles
}

#[cfg(unix)] if has_vdso_sgx_enter_enclave() {catch_sighup()};
#[cfg(unix)]
if has_vdso_sgx_enter_enclave() {
catch_sighup()
};

let (io_queue_send, io_queue_receive) = tokio::sync::mpsc::unbounded_channel();

Expand All @@ -1009,10 +1013,10 @@ impl EnclaveState {
#[cfg(unix)]
if has_vdso_sgx_enter_enclave() {
// * The enclave thread may be in a long-running `AEX/enclu[ERESUME]` loop.
// Issuing a signal to return execution control back to the enclave-runner's worker thread.
// * In non-vdso case execution control is claimed during AEX using special handler provided by AEP address
// in coenter() function and signal is not required
// for vdso case: execution control back to the enclave-runner worker thread returned
// by means of issued signal and rewriting IP in signal handler.
unsafe { libc::pthread_kill(handler.as_pthread_t() as _, signal::SIGHUP as _); }
// for non-vdso case: execution control returned using AEP set next after Enclu to break AEX/enclu[EResume] loop (signal not required).
}
let _ = handler.join();
}
Expand Down Expand Up @@ -1238,17 +1242,20 @@ extern "C" fn handle_trap(_signo: c_int, _info: *mut siginfo_t, context: *mut c_
#[cfg(unix)]
fn catch_sighup() {
unsafe {
extern "C" fn handle_sighup(_signo: c_int, _info: *mut siginfo_t, _context: *mut c_void) {
extern "C" fn handle_sighup(_signo: c_int, _info: *mut siginfo_t, context: *mut c_void) {
eprintln!("SIGHUP triggered, thread_id: {:?}", std::thread::current().id());
let instruction_ptr = unsafe { (*(_context as *mut ucontext_t)).uc_mcontext.gregs[REG_RIP as usize] as *const u8};
let instruction_ptr = unsafe { (*(context as *mut ucontext_t)).uc_mcontext.gregs[REG_RIP as usize] as *const u8};
// enclu instruction code
const ENCLU: [u8; 3] = [0x0f, 0x01, 0xd7];
let is_enclu = ENCLU.iter().enumerate().all(|(idx, v)| {
unsafe { *instruction_ptr.offset(idx as isize) == *v }
});
if is_enclu {
// Interrupt enclave execution by setting IP to the instruction following the ENCLU to mimic normal ENCLU[EXIT])
unsafe { (*(_context as *mut ucontext_t)).uc_mcontext.gregs[REG_RIP as usize] += 3 }
if is_enclu && EXITING.with(|cell| cell.borrow().as_ref().is_some_and(|val| val.load(Ordering::SeqCst) == true)) {
// Interrupt enclave execution by setting IP to the instruction following the ENCLU to mimic normal ENCLU[EEXIT])
unsafe {
(*(context as *mut ucontext_t)).uc_mcontext.gregs[REG_RIP as usize] += ENCLU.len() as i64;
(*(context as *mut ucontext_t)).uc_mcontext.gregs[REG_RAX as usize] = Enclu::EExit as i64;
}
}
let _ = stderr().flush();
}
Expand Down

0 comments on commit 3b3ad1a

Please sign in to comment.