[PAL/Linux-SGX] Prepare AEX handler to call C code in release mode

Previously, the AEX handler only called C code (more specific to our use
case, the code that works on the untrusted-thread stack) for SGX
profiling logic and only when Gramine was built in debug mode.

As a preparation for AEX Notify support, this commit modifies the AEX
handler such that it can call C code when Gramine is built in release
mode. Also, the AEX handler logic is not restricted to SGX profiling
only, so the `_PROF`/`_prof` suffixes in related function and variable
names are removed.

This commit also adds two small prerequisites for AEX Notify: (1)
`eenter_pointer` helper so that signal handling logic can learn whether
an exception happened at EENTER instruction, and (2) dummy function
`maybe_raise_pending_signal` that will be populated in a later commit.

Signed-off-by: Dmitrii Kuvaiskii <[email protected]>

Author: Dmitrii Kuvaiskii

pal/src/host/linux-sgx/enclave_framework.c

@@ -63,27 +63,19 @@ bool sgx_is_valid_untrusted_ptr(const void* _addr, size_t size, size_t alignment
 }
 
 /*
- * When DEBUG is enabled, we run sgx_profile_sample() during asynchronous enclave exit (AEX), which
+ * We run some functions (e.g. sgx_profile_sample()) during asynchronous enclave exit (AEX), which
  * uses the stack. Make sure to update URSP so that the AEX handler does not overwrite the part of
  * the stack that we just allocated.
  *
  * (Recall that URSP is an outside stack pointer, saved by EENTER and restored on AEX by the SGX
  * hardware itself.)
  */
-#ifdef DEBUG
-
 #define UPDATE_USTACK(_ustack)                           \
     do {                                                 \
         SET_ENCLAVE_TCB(ustack, _ustack);                \
         GET_ENCLAVE_TCB(gpr)->ursp = (uint64_t)_ustack;  \
     } while(0)
 
-#else
-
-#define UPDATE_USTACK(_ustack) SET_ENCLAVE_TCB(ustack, _ustack)
-
-#endif
-
 void* sgx_prepare_ustack(void) {
     void* old_ustack = GET_ENCLAVE_TCB(ustack);
 

pal/src/host/linux-sgx/generated_offsets.c

@@ -106,7 +106,7 @@ const struct generated_offset generated_offsets[] = {
 
     /* struct pal_host_tcb aka PAL_HOST_TCB */
     OFFSET(PAL_HOST_TCB_TCS, pal_host_tcb, tcs),
-    OFFSET(PAL_HOST_TCB_IN_AEX_PROF, pal_host_tcb, is_in_aex_profiling),
+    OFFSET(PAL_HOST_TCB_IN_AEX, pal_host_tcb, is_in_aex),
     OFFSET(PAL_HOST_TCB_EENTER_CNT, pal_host_tcb, eenter_cnt),
     OFFSET(PAL_HOST_TCB_EEXIT_CNT, pal_host_tcb, eexit_cnt),
     OFFSET(PAL_HOST_TCB_AEX_CNT, pal_host_tcb, aex_cnt),

pal/src/host/linux-sgx/host_entry.S

@@ -14,9 +14,8 @@
 
 #include "asm-offsets.h"
 
-    .extern tcs_base
-    .extern g_in_aex_profiling
     .extern maybe_dump_and_reset_stats
+    .extern maybe_raise_pending_signal
 
     .global sgx_ecall
     .type sgx_ecall, @function
@@ -54,9 +53,14 @@ sgx_ecall:
     leaq async_exit_pointer(%rip), %rcx
 
     movq $EENTER, %rax
+
+    .global eenter_pointer
+    .type eenter_pointer, @function
+
+eenter_pointer:
     enclu
 
-    # currently only ECALL_THREAD_RESET returns
+    # currently only ECALL_THREAD_RESET ecall returns, as well as stage-1 signal handler
 .Lafter_resume:
     popq %r15
     .cfi_adjust_cfa_offset -8
@@ -80,12 +84,15 @@ async_exit_pointer:
     .cfi_startproc
     .cfi_undefined %rip
 
+    # Inform that we are in AEX code
+    movb $1, %gs:PAL_HOST_TCB_IN_AEX
+
     # increment per-thread AEX counter for stats
     lock incq %gs:PAL_HOST_TCB_AEX_CNT
 
-#ifdef DEBUG
-    # Inform that we are in AEX profiling code
-    movb $1, %gs:PAL_HOST_TCB_IN_AEX_PROF
+    subq $RED_ZONE_SIZE, %rsp
+    .cfi_adjust_cfa_offset RED_ZONE_SIZE
+
     # Save ERESUME parameters
     pushq %rax
     .cfi_adjust_cfa_offset 8
@@ -99,11 +106,15 @@ async_exit_pointer:
     .cfi_def_cfa_register %rbp
     andq $~0xF, %rsp
 
+#ifdef DEBUG
     # Call sgx_profile_sample_aex with %rdi = TCS
     movq %rbx, %rdi
     call sgx_profile_sample_aex
 
     call maybe_dump_and_reset_stats
+#endif
+
+    call maybe_raise_pending_signal
 
     # Restore stack
     movq %rbp, %rsp
@@ -116,9 +127,11 @@ async_exit_pointer:
     .cfi_adjust_cfa_offset -8
     popq %rax
     .cfi_adjust_cfa_offset -8
-    movb $0, %gs:PAL_HOST_TCB_IN_AEX_PROF
-#endif
 
+    addq $RED_ZONE_SIZE, %rsp
+    .cfi_adjust_cfa_offset -RED_ZONE_SIZE
+
+    movb $0, %gs:PAL_HOST_TCB_IN_AEX
     .cfi_endproc
 
     # fall-through to ERESUME

pal/src/host/linux-sgx/host_exception.c

@@ -88,8 +88,8 @@ static bool interrupted_in_enclave(struct ucontext* uc) {
     return rip >= (unsigned long)async_exit_pointer && rip < (unsigned long)async_exit_pointer_end;
 }
 
-static bool interrupted_in_aex_profiling(void) {
-    return pal_get_host_tcb()->is_in_aex_profiling != 0;
+static bool interrupted_in_aex(void) {
+    return pal_get_host_tcb()->is_in_aex != 0;
 }
 
 static void handle_sync_signal(int signum, siginfo_t* info, struct ucontext* uc) {
@@ -153,7 +153,7 @@ static void handle_async_signal(int signum, siginfo_t* info, struct ucontext* uc
         for (size_t i = 0; i < g_rpc_queue->rpc_threads_cnt; i++)
             DO_SYSCALL(tkill, g_rpc_queue->rpc_threads[i], SIGUSR2);
 
-    if (interrupted_in_enclave(uc) || interrupted_in_aex_profiling()) {
+    if (interrupted_in_enclave(uc) || interrupted_in_aex()) {
         /* signal arrived while in app/LibOS/trusted PAL code or when handling another AEX, handle
          * signal inside enclave */
         pal_get_host_tcb()->async_signal_cnt++;
@@ -287,3 +287,7 @@ void maybe_dump_and_reset_stats(void) {
     }
 }
 #endif
+
+void maybe_raise_pending_signal(void) {
+    /* TODO: check if there is any sync or async pending signal and raise it */
+}

pal/src/host/linux-sgx/host_internal.h

@@ -114,6 +114,7 @@ int sgx_ecall(long ecall_no, void* ms);
 int sgx_raise(int event);
 
 void async_exit_pointer(void);
+void eenter_pointer(void);
 void eresume_pointer(void);
 void async_exit_pointer_end(void);
 

pal/src/host/linux-sgx/pal_tcb.h

@@ -92,7 +92,7 @@ typedef struct pal_host_tcb {
     sgx_arch_tcs_t* tcs;           /* TCS page of SGX corresponding to thread, for EENTER */
     void* stack;                   /* bottom of stack, for later freeing when thread exits */
     void* alt_stack;               /* bottom of alt stack, for child thread to init alt stack */
-    uint8_t is_in_aex_profiling;   /* non-zero if thread is currently doing AEX profiling */
+    uint8_t is_in_aex;             /* non-zero if thread is currently running AEX code */
     atomic_ulong eenter_cnt;       /* # of EENTERs, corresponds to # of ECALLs */
     atomic_ulong eexit_cnt;        /* # of EEXITs, corresponds to # of OCALLs */
     atomic_ulong aex_cnt;          /* # of AEXs, corresponds to # of interrupts/signals */
@@ -122,4 +122,6 @@ void collect_and_print_sgx_stats(void);
 void maybe_dump_and_reset_stats(void);
 #endif /* DEBUG */
 
+void maybe_raise_pending_signal(void);
+
 #endif /* IN_ENCLAVE */