diff --git a/base/c.jl b/base/c.jl index 3b32f683d4965..00181e9c20b2b 100644 --- a/base/c.jl +++ b/base/c.jl @@ -202,8 +202,20 @@ end # within a long-running C routine. sigatomic_begin() = ccall(:jl_sigatomic_begin, Void, ()) sigatomic_end() = ccall(:jl_sigatomic_end, Void, ()) -disable_sigint(f::Function) = try sigatomic_begin(); f(); finally sigatomic_end(); end -reenable_sigint(f::Function) = try sigatomic_end(); f(); finally sigatomic_begin(); end +function disable_sigint(f::Function) + sigatomic_begin() + res = f() + # Exception unwind sigatomic automatically + sigatomic_end() + res +end +function reenable_sigint(f::Function) + sigatomic_end() + res = f() + # Exception unwind sigatomic automatically + sigatomic_begin() + res +end function ccallable(f::Function, rt::Type, argt::Type, name::Union{AbstractString,Symbol}=string(f)) ccall(:jl_extern_c, Void, (Any, Any, Any, Cstring), f, rt, argt, name) diff --git a/base/inference.jl b/base/inference.jl index 3cae5599d0205..71a85b6e08ef0 100644 --- a/base/inference.jl +++ b/base/inference.jl @@ -1562,6 +1562,7 @@ function typeinf_loop(frame) frame.inworkq || typeinf_frame(frame) return end + ccall(:jl_sigatomic_begin, Void, ()) try in_typeinf_loop = true # the core type-inference algorithm @@ -1608,6 +1609,7 @@ function typeinf_loop(frame) println(ex) ccall(:jlbacktrace, Void, ()) end + ccall(:jl_sigatomic_end, Void, ()) nothing end diff --git a/src/Makefile b/src/Makefile index 9589bf3bd5ed9..28e4022716686 100644 --- a/src/Makefile +++ b/src/Makefile @@ -28,7 +28,7 @@ SRCS := \ jltypes gf typemap ast builtins module interpreter \ alloc dlload sys init task array dump toplevel jl_uv jlapi signal-handling \ simplevector APInt-C runtime_intrinsics runtime_ccall \ - threadgroup threading stackwalk gc + threadgroup threading stackwalk gc safepoint ifeq ($(JULIACODEGEN),LLVM) SRCS += codegen disasm debuginfo llvm-simdloop llvm-gcroot diff --git a/src/ast.c b/src/ast.c index f68ee35f730fa..a22e0b3a74c2f 100644 --- a/src/ast.c +++ b/src/ast.c @@ -230,6 +230,7 @@ static jl_ast_context_list_t *jl_ast_ctx_freed = NULL; static jl_ast_context_t *jl_ast_ctx_enter(void) { + JL_SIGATOMIC_BEGIN(); JL_LOCK_NOGC(&flisp_lock); jl_ast_context_list_t *node; jl_ast_context_t *ctx; @@ -267,6 +268,7 @@ static jl_ast_context_t *jl_ast_ctx_enter(void) static void jl_ast_ctx_leave(jl_ast_context_t *ctx) { + JL_SIGATOMIC_END(); if (--ctx->ref) return; JL_LOCK_NOGC(&flisp_lock); @@ -285,6 +287,8 @@ void jl_init_frontend(void) jl_ast_main_ctx.task = jl_current_task; jl_ast_context_list_insert(&jl_ast_ctx_using, &jl_ast_main_ctx.list); jl_init_ast_ctx(&jl_ast_main_ctx); + // To match the one in jl_ast_ctx_leave + JL_SIGATOMIC_BEGIN(); jl_ast_ctx_leave(&jl_ast_main_ctx); } @@ -758,6 +762,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len, form = scm_to_julia(fl_ctx, expansion, 0); jl_sym_t *head = NULL; if (jl_is_expr(form)) head = ((jl_expr_t*)form)->head; + JL_SIGATOMIC_END(); if (head == jl_incomplete_sym) jl_errorf("syntax: %s", jl_string_data(jl_exprarg(form,0))); else if (head == error_sym) @@ -766,6 +771,7 @@ jl_value_t *jl_parse_eval_all(const char *fname, size_t len, jl_lineno = jl_unbox_long(jl_exprarg(form,0)); else result = jl_toplevel_eval_flex(form, 1); + JL_SIGATOMIC_BEGIN(); ast = cdr_(ast); } } diff --git a/src/builtins.c b/src/builtins.c index 71fc4f17eb8f4..a8be30b74dcef 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -194,25 +194,25 @@ JL_CALLABLE(jl_f_throw) JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh) { - JL_SIGATOMIC_BEGIN(); + // Must have no safepoint eh->prev = jl_current_task->eh; eh->gcstack = jl_pgcstack; #ifdef JULIA_ENABLE_THREADING eh->gc_state = jl_gc_state(); eh->locks_len = jl_current_task->locks.len; #endif + eh->defer_signal = jl_get_ptls_states()->defer_signal; jl_current_task->eh = eh; - // TODO: this should really go after setjmp(). see comment in - // ctx_switch in task.c. - JL_SIGATOMIC_END(); } JL_DLLEXPORT void jl_pop_handler(int n) { - while (n > 0) { - jl_eh_restore_state(jl_current_task->eh); - n--; - } + if (__unlikely(n <= 0)) + return; + jl_handler_t *eh = jl_current_task->eh; + while (--n > 0) + eh = eh->prev; + jl_eh_restore_state(eh); } // primitives ----------------------------------------------------------------- diff --git a/src/ccall.cpp b/src/ccall.cpp index 196ec82ee802f..adcc73c23c82e 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1243,28 +1243,54 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(!isVa); assert(nargt == 0); JL_GC_POP(); -#ifdef JULIA_ENABLE_THREADING -#ifdef LLVM39 - builder.CreateFence(AtomicOrdering::SequentiallyConsistent, SingleThread); -#else - builder.CreateFence(SequentiallyConsistent, SingleThread); -#endif - Value *addr; - if (imaging_mode) { - assert(ctx->signalPage); - addr = ctx->signalPage; - } - else { - addr = builder.CreateIntToPtr( - ConstantInt::get(T_size, (uintptr_t)jl_gc_signal_page), T_pint8); - } - builder.CreateLoad(addr, true); -#ifdef LLVM39 - builder.CreateFence(AtomicOrdering::SequentiallyConsistent, SingleThread); -#else - builder.CreateFence(SequentiallyConsistent, SingleThread); -#endif -#endif + emit_signal_fence(); + builder.CreateLoad(ctx->signalPage, true); + emit_signal_fence(); + return ghostValue(jl_void_type); + } + if (fptr == &jl_sigatomic_begin || + ((!f_lib || (intptr_t)f_lib == 2) && f_name && + strcmp(f_name, "jl_sigatomic_begin") == 0)) { + assert(lrt == T_void); + assert(!isVa); + assert(nargt == 0); + JL_GC_POP(); + Value *pdefer_sig = emit_defer_signal(ctx); + Value *defer_sig = builder.CreateLoad(pdefer_sig); + defer_sig = builder.CreateAdd(defer_sig, + ConstantInt::get(T_sigatomic, 1)); + builder.CreateStore(defer_sig, pdefer_sig); + emit_signal_fence(); + return ghostValue(jl_void_type); + } + if (fptr == &jl_sigatomic_end || + ((!f_lib || (intptr_t)f_lib == 2) && f_name && + strcmp(f_name, "jl_sigatomic_end") == 0)) { + assert(lrt == T_void); + assert(!isVa); + assert(nargt == 0); + JL_GC_POP(); + Value *pdefer_sig = emit_defer_signal(ctx); + Value *defer_sig = builder.CreateLoad(pdefer_sig); + emit_signal_fence(); + error_unless(builder.CreateICmpNE(defer_sig, + ConstantInt::get(T_sigatomic, 0)), + "sigatomic_end called in non-sigatomic region", ctx); + defer_sig = builder.CreateSub(defer_sig, + ConstantInt::get(T_sigatomic, 1)); + builder.CreateStore(defer_sig, pdefer_sig); + BasicBlock *checkBB = BasicBlock::Create(jl_LLVMContext, "check", + ctx->f); + BasicBlock *contBB = BasicBlock::Create(jl_LLVMContext, "cont"); + builder.CreateCondBr( + builder.CreateICmpEQ(defer_sig, ConstantInt::get(T_sigatomic, 0)), + checkBB, contBB); + builder.SetInsertPoint(checkBB); + builder.CreateLoad(builder.CreateConstGEP1_32(ctx->signalPage, -1), + true); + builder.CreateBr(contBB); + ctx->f->getBasicBlockList().push_back(contBB); + builder.SetInsertPoint(contBB); return ghostValue(jl_void_type); } if (fptr == (void(*)(void))&jl_is_leaf_type || diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 021b811b7d195..5a9cbfa66fa6a 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -455,7 +455,7 @@ static bool deserves_sret(jl_value_t *dt, Type *T) static Value *emit_nthptr_addr(Value *v, ssize_t n) { return builder.CreateGEP(builder.CreateBitCast(v, T_ppjlvalue), - ConstantInt::get(T_size, (ssize_t)n)); + ConstantInt::get(T_size, n)); } static Value *emit_nthptr_addr(Value *v, Value *idx) @@ -477,6 +477,13 @@ static Value *emit_nthptr_recast(Value *v, Value *idx, MDNode *tbaa, Type *ptype return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); } +static Value *emit_nthptr_recast(Value *v, ssize_t n, MDNode *tbaa, Type *ptype) +{ + // p = (jl_value_t**)v; *(ptype)&p[n] + Value *vptr = emit_nthptr_addr(v, n); + return tbaa_decorate(tbaa,builder.CreateLoad(builder.CreateBitCast(vptr,ptype), false)); +} + static Value *emit_typeptr_addr(Value *p) { ssize_t offset = (sizeof(jl_taggedvalue_t) - @@ -1710,3 +1717,20 @@ static Value *emit_exc_in_transit(jl_codectx_t *ctx) Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, exception_in_transit) / sizeof(void*)); return builder.CreateGEP(pexc_in_transit, ArrayRef(offset), "jl_exception_in_transit"); } + +static void emit_signal_fence(void) +{ +#ifdef LLVM39 + builder.CreateFence(AtomicOrdering::SequentiallyConsistent, SingleThread); +#else + builder.CreateFence(SequentiallyConsistent, SingleThread); +#endif +} + +static Value *emit_defer_signal(jl_codectx_t *ctx) +{ + Value *ptls = builder.CreateBitCast(ctx->ptlsStates, + PointerType::get(T_sigatomic, 0)); + Constant *offset = ConstantInt::getSigned(T_int32, offsetof(jl_tls_states_t, defer_signal) / sizeof(sig_atomic_t)); + return builder.CreateGEP(ptls, ArrayRef(offset), "jl_defer_signal"); +} diff --git a/src/codegen.cpp b/src/codegen.cpp index 68362e76dff3d..c20ad6a981311 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -246,6 +246,7 @@ static IntegerType *T_uint64; static IntegerType *T_char; static IntegerType *T_size; +static IntegerType *T_sigatomic; static Type *T_float16; static Type *T_float32; @@ -342,8 +343,6 @@ static GlobalVariable *jltls_states_var; // Imaging mode only static GlobalVariable *jltls_states_func_ptr = NULL; static size_t jltls_states_func_idx = 0; -static GlobalVariable *jl_gc_signal_page_ptr = NULL; -static size_t jl_gc_signal_page_idx = 0; #endif // important functions @@ -576,9 +575,7 @@ typedef struct { std::vector inbounds; CallInst *ptlsStates; -#ifdef JULIA_ENABLE_THREADING Value *signalPage; -#endif llvm::DIBuilder *dbuilder; bool debug_enabled; @@ -1340,6 +1337,7 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) return (jl_value_t*)jl_pchar_to_array((char*)fptr, symsize); } + int8_t gc_state = jl_gc_safe_enter(); jl_dump_asm_internal(fptr, symsize, slide, #ifndef USE_MCJIT context, @@ -1355,6 +1353,7 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) #ifndef LLVM37 fstream.flush(); #endif + jl_gc_safe_leave(gc_state); return jl_cstr_to_string(const_cast(stream.str().c_str())); } @@ -3429,12 +3428,9 @@ static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx) { // allocate a placeholder gc instruction ctx->ptlsStates = builder.CreateCall(prepare_call(jltls_states_func)); -#ifdef JULIA_ENABLE_THREADING - if (imaging_mode) { - ctx->signalPage = - tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global(jl_gc_signal_page_ptr))); - } -#endif + int nthfield = offsetof(jl_tls_states_t, safepoint) / sizeof(void*); + ctx->signalPage = emit_nthptr_recast(ctx->ptlsStates, nthfield, tbaa_const, + PointerType::get(T_psize, 0)); } void jl_codegen_finalize_temp_arg(CallInst *ptlsStates, Type *T_pjlvalue, @@ -4975,6 +4971,7 @@ static void init_julia_llvm_env(Module *m) T_size = T_uint64; else T_size = T_uint32; + T_sigatomic = Type::getIntNTy(jl_LLVMContext, sizeof(sig_atomic_t) * 8); T_psize = PointerType::get(T_size, 0); T_float16 = Type::getHalfTy(jl_LLVMContext); T_float32 = Type::getFloatTy(jl_LLVMContext); @@ -5165,10 +5162,6 @@ static void init_julia_llvm_env(Module *m) jl_emit_sysimg_slot(m, pfunctype, "jl_get_ptls_states.ptr", (uintptr_t)jl_get_ptls_states_getter(), jltls_states_func_idx); - jl_gc_signal_page_ptr = - jl_emit_sysimg_slot(m, T_pint8, "jl_gc_signal_page.ptr", - (uintptr_t)jl_gc_signal_page, - jl_gc_signal_page_idx); } #endif diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index cf26cbea7f245..a4e957bdfe93f 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -70,6 +70,11 @@ extern ExecutionEngine *jl_ExecutionEngine; typedef object::SymbolRef SymRef; #endif +// Any function that acquires this lock must be either a unmanaged thread +// or in the GC safe region and must NOT allocate anything through the GC +// while holding this lock. +// Certain functions in this file might be called from an unmanaged thread +// and cannot have any interaction with the julia runtime static uv_rwlock_t threadsafe; extern "C" void jl_init_debuginfo() @@ -128,6 +133,7 @@ extern "C" EXCEPTION_DISPOSITION _seh_exception_handler(PEXCEPTION_RECORD Except static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnname, uint8_t *Section, size_t Allocated, uint8_t *UnwindData) { + // GC safe DWORD mod_size = 0; #if defined(_CPU_X86_64_) #if !defined(USE_MCJIT) @@ -244,9 +250,10 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyFunctionEmitted(const Function &F, void *Code, size_t Size, const EmittedFunctionDetails &Details) { + // This function modify linfo->fptr in GC safe region. + // This should be fine since the GC won't scan this field. int8_t gc_state = jl_gc_safe_enter(); uv_rwlock_wrlock(&threadsafe); - jl_gc_safe_leave(gc_state); StringRef sName = F.getName(); StringMap::iterator linfo_it = linfo_in_flight.find(sName); jl_lambda_info_t *linfo = NULL; @@ -266,13 +273,12 @@ class JuliaJITEventListener: public JITEventListener const_cast(&F)->deleteBody(); #endif uv_rwlock_wrunlock(&threadsafe); + jl_gc_safe_leave(gc_state); } std::map& getMap() { - int8_t gc_state = jl_gc_safe_enter(); uv_rwlock_rdlock(&threadsafe); - jl_gc_safe_leave(gc_state); return info; } #endif // ifndef USE_MCJIT @@ -301,9 +307,10 @@ class JuliaJITEventListener: public JITEventListener virtual void NotifyObjectEmitted(const ObjectImage &obj) #endif { + // This function modify linfo->fptr in GC safe region. + // This should be fine since the GC won't scan this field. int8_t gc_state = jl_gc_safe_enter(); uv_rwlock_wrlock(&threadsafe); - jl_gc_safe_leave(gc_state); #ifdef LLVM36 object::section_iterator Section = debugObj.section_begin(); object::section_iterator EndSection = debugObj.section_end(); @@ -563,6 +570,7 @@ class JuliaJITEventListener: public JITEventListener #endif #endif uv_rwlock_wrunlock(&threadsafe); + jl_gc_safe_leave(gc_state); } // must implement if we ever start freeing code @@ -571,9 +579,7 @@ class JuliaJITEventListener: public JITEventListener std::map& getObjectMap() { - int8_t gc_state = jl_gc_safe_enter(); uv_rwlock_rdlock(&threadsafe); - jl_gc_safe_leave(gc_state); return objectmap; } #endif // USE_MCJIT @@ -1103,12 +1109,17 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int64_t *se extern "C" JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr) { + // Used by Gallium.jl const object::ObjectFile *object = NULL; DIContext *context; int64_t slide, section_slide; + int8_t gc_state = jl_gc_safe_enter(); if (!jl_DI_for_fptr(fptr, NULL, &slide, NULL, &object, NULL)) - if (!jl_dylib_DI_for_fptr(fptr, &object, &context, &slide, §ion_slide, false, NULL, NULL, NULL, NULL)) + if (!jl_dylib_DI_for_fptr(fptr, &object, &context, &slide, §ion_slide, false, NULL, NULL, NULL, NULL)) { + jl_gc_safe_leave(gc_state); return jl_nothing; + } + jl_gc_safe_leave(gc_state); if (object == NULL) return jl_nothing; return (jl_value_t*)jl_ptr_to_array_1d((jl_value_t*)jl_array_uint8_type, @@ -1119,6 +1130,8 @@ JL_DLLEXPORT jl_value_t *jl_get_dobj_data(uint64_t fptr) extern "C" JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr) { + // Used by Gallium.jl + int8_t gc_state = jl_gc_safe_enter(); std::map &objmap = jl_jit_events->getObjectMap(); std::map::iterator fit = objmap.lower_bound(fptr); @@ -1134,6 +1147,7 @@ JL_DLLEXPORT uint64_t jl_get_section_start(uint64_t fptr) } } uv_rwlock_rdunlock(&threadsafe); + jl_gc_safe_leave(gc_state); return ret; } @@ -1660,9 +1674,7 @@ void RTDyldMemoryManagerUnix::registerEHFrames(uint8_t *Addr, di->start_ip = start_ip; di->end_ip = end_ip; - JL_SIGATOMIC_BEGIN(); _U_dyn_register(di); - JL_SIGATOMIC_END(); } void RTDyldMemoryManagerUnix::deregisterEHFrames(uint8_t *Addr, @@ -1688,6 +1700,7 @@ RTDyldMemoryManager* createRTDyldMemoryManagerUnix() extern "C" uint64_t jl_getUnwindInfo(uint64_t dwAddr) { + // Might be called from unmanaged thread std::map &objmap = jl_jit_events->getObjectMap(); std::map::iterator it = objmap.lower_bound(dwAddr); uint64_t ipstart = 0; // ip of the start of the section (if found) @@ -1701,6 +1714,7 @@ uint64_t jl_getUnwindInfo(uint64_t dwAddr) extern "C" uint64_t jl_getUnwindInfo(uint64_t dwAddr) { + // Might be called from unmanaged thread std::map &info = jl_jit_events->getMap(); std::map::iterator it = info.lower_bound(dwAddr); uint64_t ipstart = 0; // ip of the first instruction in the function (if found) diff --git a/src/disasm.cpp b/src/disasm.cpp index be64c9a0d08fa..6ded32a556bbc 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -294,6 +294,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #endif ) { + // GC safe // Get the host information std::string TripleName; if (TripleName.empty()) diff --git a/src/dump.c b/src/dump.c index a1ebfede658a3..97ed96080ebad 100644 --- a/src/dump.c +++ b/src/dump.c @@ -231,9 +231,6 @@ static int jl_load_sysimg_so(void) "jl_ptls_states_getter_idx"); *sysimg_gvars[tls_getter_idx - 1] = (jl_value_t*)jl_get_ptls_states_getter(); - size_t signal_page_idx = *(size_t*)jl_dlsym(jl_sysimg_handle, - "jl_gc_signal_page_idx"); - *sysimg_gvars[signal_page_idx - 1] = (jl_value_t*)jl_gc_signal_page; #endif const char *cpu_target = (const char*)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_target"); if (strcmp(cpu_target,jl_options.cpu_target) != 0) diff --git a/src/gc.c b/src/gc.c index 2d7352e39ecda..a1d0851a959b4 100644 --- a/src/gc.c +++ b/src/gc.c @@ -45,7 +45,8 @@ static jl_mutex_t finalizers_lock; * threads that enters `jl_gc_collect()` at the same time (or later calling * from unmanaged code) will wait in `jl_gc_collect()` until the GC is finished. * - * Before starting the mark phase the GC thread calls `jl_gc_signal_begin()` + * Before starting the mark phase the GC thread calls `jl_safepoint_gc_start()` + * and `jl_gc_wait_for_the_world()` * to make sure all the thread are in a safe state for the GC. The function * activates the safepoint and wait for all the threads to get ready for the * GC (`gc_state != 0`). It also acquires the `finalizers` lock so that no @@ -354,95 +355,26 @@ NOINLINE static uintptr_t gc_get_stack_ptr(void) #include "gc-debug.c" -// Only one thread can be doing the collection right now. That thread set -// `jl_running_gc` to one on entering the GC and set it back afterward. -static volatile uint64_t jl_gc_running = 0; - #ifdef JULIA_ENABLE_THREADING -JL_DLLEXPORT volatile size_t *jl_gc_signal_page = NULL; - -static void jl_wait_for_gc(void) -{ - while (jl_gc_running) { - jl_cpu_pause(); // yield? - } -} - -void jl_gc_signal_wait(void) -{ - int8_t state = jl_gc_state(); - jl_get_ptls_states()->gc_state = JL_GC_STATE_WAITING; - jl_wait_for_gc(); - jl_get_ptls_states()->gc_state = state; -} - static void jl_gc_wait_for_the_world(void) { for (int i = 0;i < jl_n_threads;i++) { jl_tls_states_t *ptls = jl_all_task_states[i].ptls; - while (!ptls->gc_state) { + // FIXME: The acquire load pairs with the release stores + // in the signal handler of safepoint so we are sure that + // all the stores on those threads are visible. However, + // we're currently not using atomic stores in mutator threads. + // We should either use atomic store release there too or use signals + // to flush the memory operations on those threads. + while (!ptls->gc_state || !jl_atomic_load_acquire(&ptls->gc_state)) { jl_cpu_pause(); // yield? } } } - -void jl_gc_signal_init(void) -{ - // jl_page_size isn't available yet. -#ifdef _OS_WINDOWS_ - jl_gc_signal_page = (size_t*)VirtualAlloc(NULL, jl_getpagesize(), - MEM_COMMIT, PAGE_READONLY); #else - jl_gc_signal_page = (size_t*)mmap(0, jl_getpagesize(), PROT_READ, - MAP_NORESERVE | MAP_PRIVATE | - MAP_ANONYMOUS, -1, 0); - if (jl_gc_signal_page == MAP_FAILED) - jl_gc_signal_page = NULL; -#endif - if (jl_gc_signal_page == NULL) { - jl_printf(JL_STDERR, "could not allocate GC synchronization page\n"); - gc_debug_critical_error(); - abort(); - } -} - -static void jl_gc_signal_begin(void) +static inline void jl_gc_wait_for_the_world(void) { -#ifdef __APPLE__ - // This needs to be after setting `jl_gc_running` so that only one thread - // can talk to the signal handler - jl_mach_gc_begin(); -#endif -#ifdef _OS_WINDOWS_ - DWORD old_prot; - VirtualProtect((void*)jl_gc_signal_page, jl_page_size, - PAGE_NOACCESS, &old_prot); -#else - mprotect((void*)jl_gc_signal_page, jl_page_size, PROT_NONE); -#endif - jl_gc_wait_for_the_world(); - JL_LOCK_NOGC(&finalizers_lock); -} - -static void jl_gc_signal_end(void) -{ - JL_UNLOCK_NOGC(&finalizers_lock); -#ifdef _OS_WINDOWS_ - DWORD old_prot; - VirtualProtect((void*)jl_gc_signal_page, jl_page_size, - PAGE_READONLY, &old_prot); -#else - mprotect((void*)jl_gc_signal_page, jl_page_size, PROT_READ); -#endif -#ifdef __APPLE__ - jl_mach_gc_end(); -#endif } -#else - -#define jl_gc_signal_begin() -#define jl_gc_signal_end() - #endif static int jl_gc_finalizers_inhibited; // don't run finalizers during codegen #11956 @@ -2326,37 +2258,27 @@ JL_DLLEXPORT void jl_gc_collect(int full) return; char *stack_hi = (char*)gc_get_stack_ptr(); gc_debug_print(); - JL_SIGATOMIC_BEGIN(); int8_t old_state = jl_gc_state(); jl_get_ptls_states()->gc_state = JL_GC_STATE_WAITING; - // In case multiple threads enter the GC at the same time, only allow - // one of them to actually run the collection. We can't just let the - // master thread do the GC since it might be running unmanaged code - // and can take arbitrarily long time before hitting a safe point. - if (jl_atomic_compare_exchange(&jl_gc_running, 0, 1) != 0) { -#ifdef JULIA_ENABLE_THREADING - JL_SIGATOMIC_END(); - jl_wait_for_gc(); + // `jl_safepoint_start_gc()` makes sure only one thread can + // run the GC. + if (!jl_safepoint_start_gc()) { + // Multithread only. See assertion in `safepoint.c` jl_gc_state_set(old_state, JL_GC_STATE_WAITING); -#else - // For single thread, GC should not call itself (in finalizers) before - // setting jl_gc_running to false so this should never happen. - assert(0 && "GC synchronization failure"); -#endif return; } - jl_gc_signal_begin(); + // no-op for non-threading + jl_gc_wait_for_the_world(); - if (!jl_gc_disable_counter) + if (!jl_gc_disable_counter) { + JL_LOCK_NOGC(&finalizers_lock); _jl_gc_collect(full, stack_hi); + JL_UNLOCK_NOGC(&finalizers_lock); + } - // Need to reset the page protection before resetting the flag since - // the thread will trigger a segfault immediately after returning from - // the signal handler. - jl_gc_signal_end(); - jl_gc_running = 0; - JL_SIGATOMIC_END(); + // no-op for non-threading + jl_safepoint_end_gc(); jl_gc_state_set(old_state, JL_GC_STATE_WAITING); if (!jl_gc_finalizers_inhibited) { diff --git a/src/init.c b/src/init.c index d8de162be8917..ed8a612a44080 100644 --- a/src/init.c +++ b/src/init.c @@ -528,16 +528,23 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel) jl_options.load = abspath(jl_options.load); } +static void jl_set_io_wait(int v) +{ + jl_get_ptls_states()->io_wait = v; +} + void _julia_init(JL_IMAGE_SEARCH rel) { #ifdef JULIA_ENABLE_THREADING // Make sure we finalize the tls callback before starting any threads. jl_get_ptls_states_getter(); - jl_gc_signal_init(); #endif + jl_safepoint_init(); libsupport_init(); + ios_set_io_wait_func = jl_set_io_wait; jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.), // best to call this first, since it also initializes libuv + jl_init_signal_async(); restore_signals(); jl_resolve_sysimg_location(rel); // loads sysimg if available, and conditionally sets jl_options.cpu_target diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 47af979595b92..82907e70e94d8 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -812,11 +812,11 @@ static void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL #endif } -#ifdef JULIA_ENABLE_THREADING // only used in the threading build +#ifdef JULIA_ENABLE_THREADING // Emit a slot in the system image to be filled at sysimg init time. // Returns the global var. Fill `idx` with 1-base index in the sysimg gv. // Use as an optimization for runtime constant addresses to have one less -// load. +// load. (Used only by threading). static GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name, uintptr_t init, size_t &idx) { @@ -915,12 +915,6 @@ static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, GlobalVariable::ExternalLinkage, ConstantInt::get(T_size, jltls_states_func_idx), "jl_ptls_states_getter_idx")); - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size, jl_gc_signal_page_idx), - "jl_gc_signal_page_idx")); #endif Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target); diff --git a/src/jl_uv.c b/src/jl_uv.c index e7169d499bfd3..1ace3d523bad9 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -47,6 +47,26 @@ extern "C" { #endif +static uv_async_t signal_async; + +static void jl_signal_async_cb(uv_async_t *hdl) +{ + // This should abort the current loop and the julia code it returns to + // or the safepoint in the callers of `uv_run` should throw the exception. + (void)hdl; + uv_stop(jl_io_loop); +} + +void jl_wake_libuv(void) +{ + uv_async_send(&signal_async); +} + +void jl_init_signal_async(void) +{ + uv_async_init(jl_io_loop, &signal_async, jl_signal_async_cb); +} + extern jl_module_t *jl_old_base_module; static jl_value_t *close_cb = NULL; @@ -80,6 +100,8 @@ JL_DLLEXPORT void jl_uv_closeHandle(uv_handle_t *handle) // also let the client app do its own cleanup if (handle->type != UV_FILE && handle->data) jl_uv_call_close_callback((jl_value_t*)handle->data); + if (handle == (uv_handle_t*)&signal_async) + return; free(handle); } @@ -114,6 +136,7 @@ JL_DLLEXPORT int jl_run_once(uv_loop_t *loop) { if (loop) { loop->stop_flag = 0; + jl_gc_safepoint(); return uv_run(loop,UV_RUN_ONCE); } else return 0; @@ -123,6 +146,7 @@ JL_DLLEXPORT void jl_run_event_loop(uv_loop_t *loop) { if (loop) { loop->stop_flag = 0; + jl_gc_safepoint(); uv_run(loop,UV_RUN_DEFAULT); } } @@ -131,6 +155,7 @@ JL_DLLEXPORT int jl_process_events(uv_loop_t *loop) { if (loop) { loop->stop_flag = 0; + jl_gc_safepoint(); return uv_run(loop,UV_RUN_NOWAIT); } else return 0; diff --git a/src/jlapi.c b/src/jlapi.c index 479132d996c59..4f4fcdf79ce55 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -229,7 +229,7 @@ JL_DLLEXPORT void jl_sigatomic_begin(void) JL_DLLEXPORT void jl_sigatomic_end(void) { - if (jl_defer_signal == 0) + if (jl_get_ptls_states()->defer_signal == 0) jl_error("sigatomic_end called in non-sigatomic region"); JL_SIGATOMIC_END(); } diff --git a/src/julia.h b/src/julia.h index e5bb7543cdc8d..ddf151873e3dc 100644 --- a/src/julia.h +++ b/src/julia.h @@ -635,6 +635,7 @@ JL_DLLEXPORT void jl_clear_malloc_data(void); // GC write barriers JL_DLLEXPORT void jl_gc_queue_root(jl_value_t *root); // root isa jl_value_t* +// Do NOT put a safepoint here STATIC_INLINE void jl_gc_wb(void *parent, void *ptr) { // parent and ptr isa jl_value_t* @@ -1392,20 +1393,16 @@ JL_DLLEXPORT void jl_yield(void); // async signal handling ------------------------------------------------------ -#include - -JL_DLLEXPORT extern volatile sig_atomic_t jl_signal_pending; -JL_DLLEXPORT extern volatile sig_atomic_t jl_defer_signal; - -#define JL_SIGATOMIC_BEGIN() jl_atomic_fetch_add(&jl_defer_signal, 1) -#define JL_SIGATOMIC_END() \ - do { \ - if (jl_atomic_fetch_add(&jl_defer_signal, -1) == 1 \ - && jl_signal_pending != 0) { \ - jl_signal_pending = 0; \ - jl_sigint_action(); \ +#define JL_SIGATOMIC_BEGIN() do { \ + jl_get_ptls_states()->defer_signal++; \ + jl_signal_fence(); \ + } while (0) +#define JL_SIGATOMIC_END() do { \ + jl_signal_fence(); \ + if (--jl_get_ptls_states()->defer_signal == 0) { \ + jl_sigint_safepoint(); \ } \ - } while(0) + } while (0) JL_DLLEXPORT void jl_sigint_action(void); JL_DLLEXPORT void jl_install_sigint_handler(void); @@ -1423,6 +1420,7 @@ typedef struct _jl_handler_t { #ifdef JULIA_ENABLE_THREADING size_t locks_len; #endif + sig_atomic_t defer_signal; } jl_handler_t; typedef struct _jl_task_t { @@ -1509,10 +1507,13 @@ static inline void jl_lock_frame_pop(void) STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh) { - JL_SIGATOMIC_BEGIN(); + // `eh` may not be `jl_current_task->eh`. See `jl_pop_handler` + // This function should **NOT** have any safepoint before the ones at the + // end. + sig_atomic_t old_defer_signal = jl_get_ptls_states()->defer_signal; + int8_t old_gc_state = jl_get_ptls_states()->gc_state; jl_current_task->eh = eh->prev; jl_pgcstack = eh->gcstack; - jl_gc_state_save_and_set(eh->gc_state); #ifdef JULIA_ENABLE_THREADING arraylist_t *locks = &jl_current_task->locks; if (locks->len > eh->locks_len) { @@ -1521,7 +1522,14 @@ STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh) locks->len = eh->locks_len; } #endif - JL_SIGATOMIC_END(); + jl_get_ptls_states()->defer_signal = eh->defer_signal; + jl_get_ptls_states()->gc_state = eh->gc_state; + if (old_gc_state && !eh->gc_state) { + jl_gc_safepoint(); + } + if (old_defer_signal && !eh->defer_signal) { + jl_sigint_safepoint(); + } } JL_DLLEXPORT void jl_enter_handler(jl_handler_t *eh); diff --git a/src/julia_internal.h b/src/julia_internal.h index 2cc304efc968b..f3f5ae33dca37 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -255,6 +255,7 @@ void jl_init_root_task(void *stack, size_t ssize); void jl_init_serializer(void); void jl_gc_init(void); void jl_init_restored_modules(jl_array_t *init_order); +void jl_init_signal_async(void); void _julia_init(JL_IMAGE_SEARCH rel); #ifdef COPY_STACKS @@ -266,10 +267,63 @@ void jl_set_base_ctx(char *__stk); void jl_init_threading(void); void jl_start_threads(void); void jl_shutdown_threading(void); + +// Whether the GC is running +extern char *jl_safepoint_pages; +STATIC_INLINE int jl_addr_is_safepoint(uintptr_t addr) +{ + uintptr_t safepoint_addr = (uintptr_t)jl_safepoint_pages; + return addr >= safepoint_addr && addr < safepoint_addr + jl_page_size * 3; +} +extern volatile uint32_t jl_gc_running; +// All the functions are safe to be called from within a signal handler +// provided that the thread will not be interrupted by another asynchronous +// signal. +// Initialize the safepoint +void jl_safepoint_init(void); +// Start the GC, return `1` if the thread should be running the GC. +// Otherwise, the thread will wait in this function until the GC finishes on +// another thread and return `0`. +// The caller should have saved the `gc_state` and set it to `WAITING` +// before calling this function. If the calling thread is to run the GC, +// it should also wait for the mutator threads to hit a safepoint **AFTER** +// this function returns +int jl_safepoint_start_gc(void); +// Can only be called by the thread that have got a `1` return value from +// `jl_safepoint_start_gc()`. This disables the safepoint (for GC, +// the `mprotect` may not be removed if there's pending SIGINT) and wake +// up waiting threads if there's any. +// The caller should restore `gc_state` **AFTER** calling this function. +void jl_safepoint_end_gc(void); +// Wait for the GC to finish +// This function does **NOT** modify the `gc_state` to inform the GC thread +// The caller should set it **BEFORE** calling this function. +void jl_safepoint_wait_gc(void); + +// Set pending sigint and enable the mechanisms to deliver the sigint. +void jl_safepoint_enable_sigint(void); +// If the safepoint is enabled to deliver sigint, disable it +// so that the thread won't repeatedly trigger it in a sigatomic region +// while not being able to actually throw the exception. +void jl_safepoint_defer_sigint(void); +// Clear the sigint pending flag and disable the mechanism to deliver sigint. +// Return `1` if the sigint should be delivered and `0` if there's no sigint +// to be delivered. +int jl_safepoint_consume_sigint(void); +void jl_wake_libuv(void); + #ifdef JULIA_ENABLE_THREADING jl_get_ptls_states_func jl_get_ptls_states_getter(void); -void jl_gc_signal_init(void); -void jl_gc_signal_wait(void); +static inline void jl_set_gc_and_wait(void) +{ + // reading own gc state doesn't need atomic ops since no one else + // should store to it. + int8_t state = jl_gc_state(); + jl_atomic_store_release(&jl_get_ptls_states()->gc_state, + JL_GC_STATE_WAITING); + jl_safepoint_wait_gc(); + jl_atomic_store_release(&jl_get_ptls_states()->gc_state, state); +} #endif void jl_dump_bitcode(char *fname, const char *sysimg_data, size_t sysimg_len); @@ -292,6 +346,7 @@ jl_value_t *skip_meta(jl_array_t *body); int has_meta(jl_array_t *body, jl_sym_t *sym); // backtraces +// Might be called from unmanaged thread uint64_t jl_getUnwindInfo(uint64_t dwBase); #ifdef _OS_WINDOWS_ #include @@ -480,11 +535,11 @@ JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a); extern jl_mutex_t typecache_lock; extern jl_mutex_t codegen_lock; +extern jl_mutex_t safepoint_lock; // -- gc.c -- // #if defined(__APPLE__) && defined(JULIA_ENABLE_THREADING) -void jl_mach_gc_begin(void); void jl_mach_gc_end(void); #endif diff --git a/src/julia_threads.h b/src/julia_threads.h index 5fc5ad8d6a104..0179f4a898c8c 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -26,12 +26,14 @@ #ifndef _OS_WINDOWS_ # include #endif +#include // This includes all the thread local states we care about for a thread. #define JL_MAX_BT_SIZE 80000 typedef struct _jl_tls_states_t { struct _jl_gcframe_t *pgcstack; struct _jl_value_t *exception_in_transit; + volatile size_t *safepoint; // Whether it is safe to execute GC at the same time. #define JL_GC_STATE_WAITING 1 // gc_state = 1 means the thread is doing GC or is waiting for the GC to @@ -42,6 +44,7 @@ typedef struct _jl_tls_states_t { volatile int8_t gc_state; volatile int8_t in_finalizer; int8_t disable_gc; + volatile sig_atomic_t defer_signal; struct _jl_thread_heap_t *heap; struct _jl_module_t *current_module; struct _jl_task_t *volatile current_task; @@ -57,6 +60,12 @@ typedef struct _jl_tls_states_t { size_t bt_size; // JL_MAX_BT_SIZE + 1 elements long uintptr_t *bt_data; + // Atomically set by the sender, reset by the handler. + volatile sig_atomic_t signal_request; + // Allow the sigint to be raised asynchronously + // this is limited to the few places we do synchronous IO + // we can make this more general (similar to defer_signal) if necessary + volatile sig_atomic_t io_wait; } jl_tls_states_t; #ifdef __MIC__ @@ -120,6 +129,8 @@ static inline unsigned long JL_CONST_FUNC jl_thread_self(void) // the __atomic builtins or c11 atomics with GNU extension or c11 _Generic # define jl_atomic_compare_exchange(obj, expected, desired) \ __sync_val_compare_and_swap(obj, expected, desired) +# define jl_atomic_exchange(obj, desired) \ + __atomic_exchange_n(obj, desired, __ATOMIC_SEQ_CST) // TODO: Maybe add jl_atomic_compare_exchange_weak for spin lock # define jl_atomic_store(obj, val) \ __atomic_store_n(obj, val, __ATOMIC_SEQ_CST) @@ -196,6 +207,31 @@ jl_atomic_compare_exchange(volatile T *obj, T2 expected, T3 desired) return (T)_InterlockedCompareExchange64((volatile __int64*)obj, (__int64)desired, (__int64)expected); } +// atomic exchange +template +static inline typename std::enable_if::type +jl_atomic_exchange(volatile T *obj, T2 val) +{ + return _InterlockedExchange8((volatile char*)obj, (char)val); +} +template +static inline typename std::enable_if::type +jl_atomic_exchange(volatile T *obj, T2 val) +{ + return _InterlockedExchange16((volatile short*)obj, (short)val); +} +template +static inline typename std::enable_if::type +jl_atomic_exchange(volatile T *obj, T2 val) +{ + return _InterlockedExchange((volatile LONG*)obj, (LONG)val); +} +template +static inline typename std::enable_if::type +jl_atomic_exchange(volatile T *obj, T2 val) +{ + return _InterlockedExchange64((volatile __int64*)obj, (__int64)val); +} // atomic stores template static inline typename std::enable_if::type @@ -258,11 +294,25 @@ JL_DLLEXPORT void (jl_cpu_wake)(void); // Accessing the tls variables, gc safepoint and gc states JL_DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void); +// This triggers a SegFault when we are in GC +// Assign it to a variable to make sure the compiler emit the load +// and to avoid Clang warning for -Wunused-volatile-lvalue +#define jl_gc_safepoint() do { \ + jl_signal_fence(); \ + size_t safepoint_load = *jl_get_ptls_states()->safepoint; \ + jl_signal_fence(); \ + (void)safepoint_load; \ + } while (0) +#define jl_sigint_safepoint() do { \ + jl_signal_fence(); \ + size_t safepoint_load = jl_get_ptls_states()->safepoint[-1]; \ + jl_signal_fence(); \ + (void)safepoint_load; \ + } while (0) #ifndef JULIA_ENABLE_THREADING extern JL_DLLEXPORT jl_tls_states_t jl_tls_states; #define jl_get_ptls_states() (&jl_tls_states) #define jl_gc_state() ((int8_t)0) -#define jl_gc_safepoint() do {} while (0) STATIC_INLINE int8_t jl_gc_state_set(int8_t state, int8_t old_state) { (void)state; @@ -273,16 +323,6 @@ typedef jl_tls_states_t *(*jl_get_ptls_states_func)(void); JL_DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f); // Make sure jl_gc_state() is always a rvalue #define jl_gc_state() ((int8_t)(jl_get_ptls_states()->gc_state)) -JL_DLLEXPORT extern volatile size_t *jl_gc_signal_page; -// This triggers a SegFault when we are in GC -// Assign it to a variable to make sure the compiler emit the load -// and to avoid Clang warning for -Wunused-volatile-lvalue -#define jl_gc_safepoint() do { \ - jl_signal_fence(); \ - size_t safepoint_load = *jl_gc_signal_page; \ - jl_signal_fence(); \ - (void)safepoint_load; \ - } while (0) STATIC_INLINE int8_t jl_gc_state_set(int8_t state, int8_t old_state) { jl_get_ptls_states()->gc_state = state; diff --git a/src/safepoint.c b/src/safepoint.c new file mode 100644 index 0000000000000..6644f32b63c61 --- /dev/null +++ b/src/safepoint.c @@ -0,0 +1,232 @@ +// This file is a part of Julia. License is MIT: http://julialang.org/license + +#include "julia.h" +#include "julia_internal.h" +#include "threading.h" +#ifndef _OS_WINDOWS_ +#include +#if defined(_OS_DARWIN_) && !defined(MAP_ANONYMOUS) +#define MAP_ANONYMOUS MAP_ANON +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// 0: no sigint is pending +// 1: at least one sigint is pending, only the sigint page is enabled. +// 2: at least one sigint is pending, both safepoint pages are enabled. +JL_DLLEXPORT sig_atomic_t jl_signal_pending = 0; +volatile uint32_t jl_gc_running = 0; +char *jl_safepoint_pages = NULL; +// The number of safepoints enabled on the three pages. +// The first page, is the SIGINT page, only used by the master thread. +// The second page, is the GC page for the master thread, this is where +// the `safepoint` tls pointer points to for the master thread. +// The third page is the GC page for the other threads. The thread's +// `safepoint` tls pointer points the beginning of this page + `sizeof(size_t)` +// so that both safepoint load and pending signal load falls in this page. +// The initialization of the `safepoint` pointer is done `ti_initthread` +// in `threading.c`. +uint8_t jl_safepoint_enable_cnt[3] = {0, 0, 0}; + +// This lock should be acquired before enabling/disabling the safepoint +// or accessing one of the following variables: +// +// * jl_gc_running +// * jl_signal_pending +// * jl_safepoint_enable_cnt +// +// Additionally accessing `jl_gc_running` should use acquire/release +// load/store so that threads waiting for the GC doesn't have to also +// fight on the safepoint lock... +// +// Acquiring and releasing this lock should use the `jl_mutex_*_nogc` functions +jl_mutex_t safepoint_lock; + +static void jl_safepoint_enable(int idx) +{ + // safepoint_lock should be held + assert(0 <= idx && idx < 3); + if (jl_safepoint_enable_cnt[idx]++ != 0) { + // We expect this to be enabled at most twice + // one for the GC, one for SIGINT. + // Update this if this is not the case anymore in the future. + assert(jl_safepoint_enable_cnt[idx] <= 2); + return; + } + // Now that we are requested to mprotect the page and it wasn't already. + char *pageaddr = jl_safepoint_pages + jl_page_size * idx; +#ifdef _OS_WINDOWS_ + DWORD old_prot; + VirtualProtect(pageaddr, jl_page_size, PAGE_NOACCESS, &old_prot); +#else + mprotect(pageaddr, jl_page_size, PROT_NONE); +#endif +} + +static void jl_safepoint_disable(int idx) +{ + // safepoint_lock should be held + assert(0 <= idx && idx < 3); + if (--jl_safepoint_enable_cnt[idx] != 0) { + assert(jl_safepoint_enable_cnt[idx] > 0); + return; + } + // Now that we are requested to un-mprotect the page and no one else + // want it to be kept protected. + char *pageaddr = jl_safepoint_pages + jl_page_size * idx; +#ifdef _OS_WINDOWS_ + DWORD old_prot; + VirtualProtect(pageaddr, jl_page_size, PAGE_READONLY, &old_prot); +#else + mprotect(pageaddr, jl_page_size, PROT_READ); +#endif +} + +void jl_safepoint_init(void) +{ + // jl_page_size isn't available yet. + size_t pgsz = jl_getpagesize(); +#ifdef _OS_WINDOWS_ + char *addr = (char*)VirtualAlloc(NULL, pgsz * 3, MEM_COMMIT, PAGE_READONLY); +#else + char *addr = (char*)mmap(0, pgsz * 3, PROT_READ, + MAP_NORESERVE | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) + addr = NULL; +#endif + if (addr == NULL) { + jl_printf(JL_STDERR, "could not allocate GC synchronization page\n"); + gc_debug_critical_error(); + abort(); + } + // The signal page is for the gc safepoint. + // The page before it is the sigint pending flag. + jl_safepoint_pages = addr; +} + +int jl_safepoint_start_gc(void) +{ +#ifdef JULIA_ENABLE_THREADING + // The thread should have set this already + assert(jl_get_ptls_states()->gc_state == JL_GC_STATE_WAITING); + jl_mutex_lock_nogc(&safepoint_lock); + // In case multiple threads enter the GC at the same time, only allow + // one of them to actually run the collection. We can't just let the + // master thread do the GC since it might be running unmanaged code + // and can take arbitrarily long time before hitting a safe point. + if (jl_atomic_compare_exchange(&jl_gc_running, 0, 1) != 0) { + jl_mutex_unlock_nogc(&safepoint_lock); + jl_safepoint_wait_gc(); + return 0; + } + jl_safepoint_enable(1); + jl_safepoint_enable(2); + jl_mutex_unlock_nogc(&safepoint_lock); + return 1; +#else + // For single thread, GC should not call itself (in finalizers) before + // setting `jl_gc_running` to false so this should never happen. + assert(!jl_gc_running); + jl_gc_running = 1; + return 1; +#endif +} + +void jl_safepoint_end_gc(void) +{ + assert(jl_gc_running); +#ifdef JULIA_ENABLE_THREADING + jl_mutex_lock_nogc(&safepoint_lock); + // Need to reset the page protection before resetting the flag since + // the thread will trigger a segfault immediately after returning from + // the signal handler. + jl_safepoint_disable(2); + jl_safepoint_disable(1); + jl_atomic_store_release(&jl_gc_running, 0); +# ifdef __APPLE__ + // This wakes up other threads on mac. + jl_mach_gc_end(); +# endif + jl_mutex_unlock_nogc(&safepoint_lock); +#else + jl_gc_running = 0; +#endif +} + +void jl_safepoint_wait_gc(void) +{ +#ifdef JULIA_ENABLE_THREADING + // The thread should have set this is already + assert(jl_get_ptls_states()->gc_state != 0); + // Use normal volatile load in the loop. + // Use a acquire load to make sure the GC result is visible on this thread. + while (jl_gc_running || jl_atomic_load_acquire(&jl_gc_running)) { + jl_cpu_pause(); // yield? + } +#else + assert(0 && "No one should wait for the GC on another thread"); +#endif +} + +void jl_safepoint_enable_sigint(void) +{ + jl_mutex_lock_nogc(&safepoint_lock); + // Make sure both safepoints are enabled exactly once for SIGINT. + switch (jl_signal_pending) { + default: + assert(0 && "Shouldn't happen."); + case 0: + // Enable SIGINT page + jl_safepoint_enable(0); + // fall through + case 1: + // SIGINT page is enabled, enable GC page + jl_safepoint_enable(1); + // fall through + case 2: + jl_signal_pending = 2; + } + jl_mutex_unlock_nogc(&safepoint_lock); +} + +void jl_safepoint_defer_sigint(void) +{ + jl_mutex_lock_nogc(&safepoint_lock); + // Make sure the GC safepoint is disabled for SIGINT. + if (jl_signal_pending == 2) { + jl_safepoint_disable(1); + jl_signal_pending = 1; + } + jl_mutex_unlock_nogc(&safepoint_lock); +} + +int jl_safepoint_consume_sigint(void) +{ + int has_signal = 0; + jl_mutex_lock_nogc(&safepoint_lock); + // Make sure both safepoints are disabled for SIGINT. + switch (jl_signal_pending) { + default: + assert(0 && "Shouldn't happen."); + case 2: + // Disable gc page + jl_safepoint_disable(1); + // fall through + case 1: + // GC page is disabled, disable SIGINT page + jl_safepoint_disable(0); + has_signal = 1; + // fall through + case 0: + jl_signal_pending = 0; + } + jl_mutex_unlock_nogc(&safepoint_lock); + return has_signal; +} + +#ifdef __cplusplus +} +#endif diff --git a/src/signal-handling.c b/src/signal-handling.c index cf92b43681fc9..3c5a97cc1a673 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -23,11 +23,35 @@ static const uint64_t GIGA = 1000000000ULL; JL_DLLEXPORT void jl_profile_stop_timer(void); JL_DLLEXPORT int jl_profile_start_timer(void); -volatile sig_atomic_t jl_signal_pending = 0; -volatile sig_atomic_t jl_defer_signal = 0; +static uint64_t jl_last_sigint_trigger = 0; +static void jl_clear_force_sigint(void) +{ + jl_last_sigint_trigger = 0; +} -int exit_on_sigint = 0; -JL_DLLEXPORT void jl_exit_on_sigint(int on) {exit_on_sigint = on;} +static int jl_check_force_sigint(void) +{ + static double accum_weight = 0; + uint64_t cur_time = uv_hrtime(); + uint64_t dt = cur_time - jl_last_sigint_trigger; + uint64_t last_t = jl_last_sigint_trigger; + jl_last_sigint_trigger = cur_time; + if (last_t == 0) { + accum_weight = 0; + return 0; + } + double new_weight = accum_weight * exp(-(dt / 1e9)) + 0.3; + if (!isnormal(new_weight)) + new_weight = 0; + accum_weight = new_weight; + return new_weight > 1; +} + +static int exit_on_sigint = 0; +JL_DLLEXPORT void jl_exit_on_sigint(int on) +{ + exit_on_sigint = on; +} // what to do on SIGINT JL_DLLEXPORT void jl_sigint_action(void) diff --git a/src/signals-mach.c b/src/signals-mach.c index 2fd7eedc2b606..8e13ed748aa20 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -18,31 +18,42 @@ static void attach_exception_port(thread_port_t thread); #ifdef JULIA_ENABLE_THREADING -static jl_mutex_t gc_suspend_lock; -// This is a copy of `jl_gc_safepoint_activated` to make it easier -// to synchronic the GC and the signal handler -static int jl_gc_safepoint_activated = 0; // low 16 bits are the thread id, the next 8 bits are the original gc_state static arraylist_t suspended_threads; -void jl_mach_gc_begin(void) -{ - JL_LOCK_NOGC(&gc_suspend_lock); - jl_gc_safepoint_activated = 1; - JL_UNLOCK_NOGC(&gc_suspend_lock); -} void jl_mach_gc_end(void) { - JL_LOCK_NOGC(&gc_suspend_lock); - jl_gc_safepoint_activated = 0; + // Requires the safepoint lock to be held for (size_t i = 0;i < suspended_threads.len;i++) { uintptr_t item = (uintptr_t)suspended_threads.items[i]; int16_t tid = (int16_t)item; int8_t gc_state = (int8_t)(item >> 8); - jl_all_task_states[tid].ptls->gc_state = gc_state; + jl_atomic_store_release(&jl_all_task_states[tid].ptls->gc_state, + gc_state); thread_resume(pthread_mach_thread_np(jl_all_task_states[tid].system_id)); } suspended_threads.len = 0; - JL_UNLOCK_NOGC(&gc_suspend_lock); +} + +// Suspend the thread and return `1` if the GC is running. +// Otherwise return `0` +static int jl_mach_gc_wait(jl_tls_states_t *ptls, + mach_port_t thread, int16_t tid) +{ + jl_mutex_lock_nogc(&safepoint_lock); + if (!jl_gc_running) { + // GC is done before we get the message or the safepoint is enabled + // for SIGINT. + jl_mutex_unlock_nogc(&safepoint_lock); + return 0; + } + // Otherwise, set the gc state of the thread, suspend and record it + int8_t gc_state = ptls->gc_state; + jl_atomic_store_release(&ptls->gc_state, JL_GC_STATE_WAITING); + uintptr_t item = tid | (((uintptr_t)gc_state) << 16); + arraylist_push(&suspended_threads, (void*)item); + thread_suspend(thread); + jl_mutex_unlock_nogc(&safepoint_lock); + return 1; } #endif @@ -176,24 +187,22 @@ kern_return_t catch_exception_raise(mach_port_t exception_port, kern_return_t ret = thread_get_state(thread, x86_EXCEPTION_STATE64, (thread_state_t)&exc_state, &exc_count); HANDLE_MACH_ERROR("thread_get_state", ret); uint64_t fault_addr = exc_state.__faultvaddr; + if (jl_addr_is_safepoint(fault_addr)) { #ifdef JULIA_ENABLE_THREADING - if (fault_addr == (uintptr_t)jl_gc_signal_page) { - JL_LOCK_NOGC(&gc_suspend_lock); - if (!jl_gc_safepoint_activated) { - // GC is done before we get the message, do nothing and return - JL_UNLOCK_NOGC(&gc_suspend_lock); + if (jl_mach_gc_wait(ptls, thread, tid)) return KERN_SUCCESS; + if (ptls->tid != 0) + return KERN_SUCCESS; +#endif + if (ptls->defer_signal) { + jl_safepoint_defer_sigint(); + } + else if (jl_safepoint_consume_sigint()) { + jl_clear_force_sigint(); + jl_throw_in_thread(tid, thread, jl_interrupt_exception); } - // Otherwise, set the gc state of the thread, suspend and record it - int8_t gc_state = ptls->gc_state; - ptls->gc_state = JL_GC_STATE_WAITING; - uintptr_t item = tid | (((uintptr_t)gc_state) << 16); - arraylist_push(&suspended_threads, (void*)item); - thread_suspend(thread); - JL_UNLOCK_NOGC(&gc_suspend_lock); return KERN_SUCCESS; } -#endif #ifdef SEGV_EXCEPTION if (1) { #else @@ -235,9 +244,8 @@ static void attach_exception_port(thread_port_t thread) HANDLE_MACH_ERROR("thread_set_exception_ports", ret); } -static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx, int sig) +static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) { - (void)sig; mach_port_t tid_port = pthread_mach_thread_np(jl_all_task_states[tid].system_id); kern_return_t ret = thread_suspend(tid_port); @@ -258,19 +266,39 @@ static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx, int si static void jl_thread_resume(int tid, int sig) { mach_port_t thread = pthread_mach_thread_np(jl_all_task_states[tid].system_id); + kern_return_t ret = thread_resume(thread); + HANDLE_MACH_ERROR("thread_resume", ret); +} - if (tid == 0 && sig == SIGINT) { - if (jl_defer_signal) { - jl_signal_pending = sig; - } - else { - jl_signal_pending = 0; - jl_throw_in_thread(tid, thread, jl_interrupt_exception); - } +// Throw jl_interrupt_exception if the master thread is in a signal async region +// or if SIGINT happens too often. +static void jl_try_deliver_sigint(void) +{ + mach_port_t thread = pthread_mach_thread_np(jl_all_task_states[0].system_id); + jl_tls_states_t *ptls = jl_all_task_states[0].ptls; + + kern_return_t ret = thread_suspend(thread); + HANDLE_MACH_ERROR("thread_suspend", ret); + + // This abort `sleep` and other syscall. + ret = thread_abort(thread); + HANDLE_MACH_ERROR("thread_abort", ret); + + jl_safepoint_enable_sigint(); + int force = jl_check_force_sigint(); + if (force || (!ptls->defer_signal && ptls->io_wait)) { + jl_safepoint_consume_sigint(); + if (force) + jl_safe_printf("WARNING: Force throwing a SIGINT\n"); + jl_clear_force_sigint(); + jl_throw_in_thread(0, thread, jl_interrupt_exception); + } + else { + jl_wake_libuv(); } - kern_return_t ret = thread_resume(thread); - HANDLE_MACH_ERROR("thread_resume", ret) + ret = thread_resume(thread); + HANDLE_MACH_ERROR("thread_resume", ret); } static int profile_started = 0; @@ -350,7 +378,7 @@ void *mach_profile_listener(void *arg) break; unw_context_t *uc; - jl_thread_suspend_and_get_state(i, &uc, -1); + jl_thread_suspend_and_get_state(i, &uc); #ifdef LIBOSXUNWIND /* diff --git a/src/signals-unix.c b/src/signals-unix.c index 8dc9a5a1c9585..98e4784ec8704 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -38,7 +38,6 @@ unsigned sig_stack_size = SIGSTKSZ; #endif static pthread_t signals_thread; -static volatile int remote_sig; static int is_addr_on_stack(jl_tls_states_t *ptls, void *addr) { @@ -89,13 +88,23 @@ static void segv_handler(int sig, siginfo_t *info, void *context) { assert(sig == SIGSEGV || sig == SIGBUS); -#ifdef JULIA_ENABLE_THREADING - if (info->si_addr == jl_gc_signal_page) { + if (jl_addr_is_safepoint((uintptr_t)info->si_addr)) { jl_unblock_signal(sig); - jl_gc_signal_wait(); +#ifdef JULIA_ENABLE_THREADING + jl_set_gc_and_wait(); + // Do not raise sigint on worker thread + if (ti_tid != 0) + return; +#endif + if (jl_get_ptls_states()->defer_signal) { + jl_safepoint_defer_sigint(); + } + else if (jl_safepoint_consume_sigint()) { + jl_clear_force_sigint(); + jl_throw(jl_interrupt_exception); + } return; } -#endif if (jl_safe_restore || is_addr_on_stack(jl_get_ptls_states(), info->si_addr)) { // stack overflow, or restarting jl_ jl_unblock_signal(sig); jl_throw(jl_stackovf_exception); @@ -131,51 +140,55 @@ static void allocate_segv_handler(void) } static unw_context_t *volatile signal_context; -static volatile int waiting_for; static pthread_mutex_t in_signal_lock; static pthread_cond_t exit_signal_cond; static pthread_cond_t signal_caught_cond; -static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx, int sig) +static void jl_thread_suspend_and_get_state(int tid, unw_context_t **ctx) { pthread_mutex_lock(&in_signal_lock); - remote_sig = sig; - waiting_for = tid; + jl_tls_states_t *ptls = jl_all_task_states[tid].ptls; + jl_atomic_store_release(&ptls->signal_request, 1); pthread_kill(jl_all_task_states[tid].system_id, SIGUSR2); pthread_cond_wait(&signal_caught_cond, &in_signal_lock); // wait for thread to acknowledge - assert(waiting_for == 0); + assert(jl_atomic_load_acquire(&ptls->signal_request) == 0); *ctx = signal_context; } static void jl_thread_resume(int tid, int sig) { (void)sig; - remote_sig = 0; - waiting_for = tid; + jl_tls_states_t *ptls = jl_all_task_states[tid].ptls; + jl_atomic_store_release(&ptls->signal_request, 1); pthread_cond_broadcast(&exit_signal_cond); pthread_cond_wait(&signal_caught_cond, &in_signal_lock); // wait for thread to acknowledge - assert(waiting_for == 0); + assert(jl_atomic_load_acquire(&ptls->signal_request) == 0); pthread_mutex_unlock(&in_signal_lock); } - -static inline void wait_barrier(void) +// Throw jl_interrupt_exception if the master thread is in a signal async region +// or if SIGINT happens too often. +static void jl_try_deliver_sigint(void) { - if (waiting_for < 0) { - if (jl_atomic_fetch_add(&waiting_for, 1) == -1) { - pthread_cond_broadcast(&signal_caught_cond); - } - } - else { - pthread_cond_broadcast(&signal_caught_cond); - waiting_for = 0; - } + jl_tls_states_t *ptls = jl_all_task_states[0].ptls; + jl_safepoint_enable_sigint(); + jl_wake_libuv(); + jl_atomic_store_release(&ptls->signal_request, 2); + // This also makes sure `sleep` is aborted. + pthread_kill(jl_all_task_states[0].system_id, SIGUSR2); } + +// request: +// 0: nothing +// 1: get state +// 3: throw sigint if `!defer_signal && io_wait` or if force throw threshold +// is reached void usr2_handler(int sig, siginfo_t *info, void *ctx) { ucontext_t *context = (ucontext_t*)ctx; - if ((remote_sig > 0 && waiting_for < 0) || waiting_for == ti_tid) { - int realsig = remote_sig; + jl_tls_states_t *ptls = jl_get_ptls_states(); + sig_atomic_t request = jl_atomic_exchange(&ptls->signal_request, 0); + if (request == 1) { #ifdef __APPLE__ signal_context = (unw_context_t*)&context->uc_mcontext->__ss; #else @@ -183,20 +196,25 @@ void usr2_handler(int sig, siginfo_t *info, void *ctx) #endif pthread_mutex_lock(&in_signal_lock); - wait_barrier(); + pthread_cond_broadcast(&signal_caught_cond); pthread_cond_wait(&exit_signal_cond, &in_signal_lock); - wait_barrier(); + request = jl_atomic_exchange(&ptls->signal_request, 0); + assert(request == 1); + (void)request; + pthread_cond_broadcast(&signal_caught_cond); pthread_mutex_unlock(&in_signal_lock); - - if (ti_tid == 0 && realsig == SIGINT) { - if (jl_defer_signal) { - jl_signal_pending = realsig; - } - else { - jl_signal_pending = 0; - jl_unblock_signal(sig); - jl_throw(jl_interrupt_exception); - } + } + else if (request == 2) { + jl_unblock_signal(sig); + int force = jl_check_force_sigint(); + if (force || (!ptls->defer_signal && ptls->io_wait)) { + jl_safepoint_consume_sigint(); + if (force) + jl_safe_printf("WARNING: Force throwing a SIGINT\n"); + // Force a throw + jl_clear_force_sigint(); + // TODO: implement `jl_throw_in_ctx` -- Jameson + jl_throw(jl_interrupt_exception); } } } @@ -351,8 +369,19 @@ static void *signal_listener(void *arg) profile = (sig == SIGUSR1); # endif #endif + if (sig == SIGINT) { + if (exit_on_sigint) { + critical = 1; + } + else { + jl_try_deliver_sigint(); + continue; + } + } + else { + critical = 0; + } - critical = (sig == SIGINT && exit_on_sigint); critical |= (sig == SIGTERM); critical |= (sig == SIGABRT); critical |= (sig == SIGQUIT); @@ -367,7 +396,7 @@ static void *signal_listener(void *arg) // (so that thread zero gets notified last) for (i = jl_n_threads; i-- > 0; ) { // notify thread to stop - jl_thread_suspend_and_get_state(i, &signal_context, sig); + jl_thread_suspend_and_get_state(i, &signal_context); // do backtrace on thread contexts for critical signals // this part must be signal-handler safe diff --git a/src/signals-win.c b/src/signals-win.c index 28aecc31108aa..8416774e65650 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -41,6 +41,22 @@ static char *strsignal(int sig) return "?"; } +static void jl_try_throw_sigint(void) +{ + jl_tls_states_t *ptls = jl_get_ptls_states(); + jl_safepoint_enable_sigint(); + jl_wake_libuv(); + int force = jl_check_force_sigint(); + if (force || (!ptls->defer_signal && ptls->io_wait)) { + jl_safepoint_consume_sigint(); + if (force) + jl_safe_printf("WARNING: Force throwing a SIGINT\n"); + // Force a throw + jl_clear_force_sigint(); + jl_throw(jl_interrupt_exception); + } +} + void __cdecl crt_sig_handler(int sig, int num) { CONTEXT Context; @@ -62,13 +78,9 @@ void __cdecl crt_sig_handler(int sig, int num) break; case SIGINT: signal(SIGINT, (void (__cdecl *)(int))crt_sig_handler); - if (jl_defer_signal) { - jl_signal_pending = sig; - } - else { - jl_signal_pending = 0; - jl_sigint_action(); - } + if (exit_on_sigint) + jl_exit(130); // 128 + SIGINT + jl_try_throw_sigint(); break; default: // SIGSEGV, (SSIGTERM, IGILL) memset(&Context, 0, sizeof(Context)); @@ -111,48 +123,60 @@ void jl_throw_in_ctx(jl_value_t *excpt, CONTEXT *ctxThread, int bt) HANDLE hMainThread = INVALID_HANDLE_VALUE; -static BOOL WINAPI sigint_handler(DWORD wsig) //This needs winapi types to guarantee __stdcall +// Try to throw the exception in the master thread. +static void jl_try_deliver_sigint(void) { - int sig; - //windows signals use different numbers from unix (raise) - switch(wsig) { - case CTRL_C_EVENT: sig = SIGINT; break; - //case CTRL_BREAK_EVENT: sig = SIGTERM; break; - // etc. - default: sig = SIGTERM; break; + jl_tls_states_t *ptls = jl_all_task_states[0].ptls; + jl_safepoint_enable_sigint(); + jl_wake_libuv(); + if ((DWORD)-1 == SuspendThread(hMainThread)) { + // error + jl_safe_printf("error: SuspendThread failed\n"); + return; } - if (jl_defer_signal) { - jl_signal_pending = sig; - } - else { - jl_signal_pending = 0; - if (exit_on_sigint) jl_exit(130); - if ((DWORD)-1 == SuspendThread(hMainThread)) { - //error - jl_safe_printf("error: SuspendThread failed\n"); - return 0; - } + int force = jl_check_force_sigint(); + if (force || (!ptls->defer_signal && ptls->io_wait)) { + jl_safepoint_consume_sigint(); + if (force) + jl_safe_printf("WARNING: Force throwing a SIGINT\n"); + // Force a throw + jl_clear_force_sigint(); CONTEXT ctxThread; - memset(&ctxThread,0,sizeof(CONTEXT)); + memset(&ctxThread, 0, sizeof(CONTEXT)); ctxThread.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; if (!GetThreadContext(hMainThread, &ctxThread)) { - //error + // error jl_safe_printf("error: GetThreadContext failed\n"); - return 0; + return; } jl_throw_in_ctx(jl_interrupt_exception, &ctxThread, 1); ctxThread.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; - if (!SetThreadContext(hMainThread,&ctxThread)) { + if (!SetThreadContext(hMainThread, &ctxThread)) { jl_safe_printf("error: SetThreadContext failed\n"); - //error - return 0; - } - if ((DWORD)-1 == ResumeThread(hMainThread)) { - jl_safe_printf("error: ResumeThread failed\n"); - //error - return 0; + // error + return; } } + if ((DWORD)-1 == ResumeThread(hMainThread)) { + jl_safe_printf("error: ResumeThread failed\n"); + // error + return; + } +} + +static BOOL WINAPI sigint_handler(DWORD wsig) //This needs winapi types to guarantee __stdcall +{ + int sig; + //windows signals use different numbers from unix (raise) + switch(wsig) { + case CTRL_C_EVENT: sig = SIGINT; break; + //case CTRL_BREAK_EVENT: sig = SIGTERM; break; + // etc. + default: sig = SIGTERM; break; + } + if (exit_on_sigint) + jl_exit(128 + sig); // 128 + SIGINT + jl_try_deliver_sigint(); return 1; } @@ -168,13 +192,23 @@ static LONG WINAPI _exception_handler(struct _EXCEPTION_POINTERS *ExceptionInfo, jl_throw_in_ctx(jl_stackovf_exception, ExceptionInfo->ContextRecord,in_ctx&&pSetThreadStackGuarantee); return EXCEPTION_CONTINUE_EXECUTION; case EXCEPTION_ACCESS_VIOLATION: + if (jl_addr_is_safepoint(ExceptionInfo->ExceptionRecord->ExceptionInformation[1])) { #ifdef JULIA_ENABLE_THREADING - if (ExceptionInfo->ExceptionRecord->ExceptionInformation[1] == - (intptr_t)jl_gc_signal_page) { - jl_gc_signal_wait(); + jl_set_gc_and_wait(); + // Do not raise sigint on worker thread + if (ti_tid != 0) + return EXCEPTION_CONTINUE_EXECUTION; +#endif + if (jl_get_ptls_states()->defer_signal) { + jl_safepoint_defer_sigint(); + } + else if (jl_safepoint_consume_sigint()) { + jl_clear_force_sigint(); + jl_throw_in_ctx(jl_interrupt_exception, + ExceptionInfo->ContextRecord, in_ctx); + } return EXCEPTION_CONTINUE_EXECUTION; } -#endif if (ExceptionInfo->ExceptionRecord->ExceptionInformation[0] == 1) { // writing to read-only memory (e.g. mmap) jl_throw_in_ctx(jl_readonlymemory_exception, ExceptionInfo->ContextRecord,in_ctx); return EXCEPTION_CONTINUE_EXECUTION; diff --git a/src/stackwalk.c b/src/stackwalk.c index e433e06f9c1dc..f1582ed831152 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -197,6 +197,7 @@ int needsSymRefreshModuleList; BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) { + // Might be called from unmanaged thread. if (needsSymRefreshModuleList && hSymRefreshModuleList != 0 && !jl_in_stackwalk) { jl_in_stackwalk = 1; hSymRefreshModuleList(GetCurrentProcess()); @@ -228,6 +229,7 @@ static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp) { + // Might be called from unmanaged thread. #ifndef _CPU_X86_64_ *ip = (uintptr_t)cursor->stackframe.AddrPC.Offset; *sp = (uintptr_t)cursor->stackframe.AddrStack.Offset; @@ -354,9 +356,11 @@ JL_DLLEXPORT jl_value_t *jl_lookup_code_address(void *ip, int skipC) size_t inlinedat_line; char *inlinedat_file; jl_lambda_info_t *outer_linfo; + int8_t gc_state = jl_gc_safe_enter(); int fromC = frame_info_from_ip(&func_name, &file_name, &line_num, &inlinedat_file, &inlinedat_line, &outer_linfo, (size_t)ip, skipC, 0); + jl_gc_safe_leave(gc_state); jl_value_t *r = (jl_value_t*)jl_alloc_svec(8); JL_GC_PUSH1(&r); jl_svecset(r, 0, jl_symbol(func_name)); diff --git a/src/support/ios.c b/src/support/ios.c index e2ed4abc8a08a..537c73d3f4e0d 100644 --- a/src/support/ios.c +++ b/src/support/ios.c @@ -36,6 +36,14 @@ extern "C" { #endif +void (*ios_set_io_wait_func)(int) = NULL; +static void set_io_wait_begin(int v) +{ + if (__likely(ios_set_io_wait_func)) { + ios_set_io_wait_func(v); + } +} + /* OS-level primitive wrappers */ #if defined(__APPLE__) || defined(_OS_WINDOWS_) @@ -92,7 +100,9 @@ static int _os_read(long fd, void *buf, size_t n, size_t *nread) ssize_t r; while (1) { + set_io_wait_begin(1); r = read((int)fd, buf, LIMIT_IO_SIZE(n)); + set_io_wait_begin(0); if (r > -1) { *nread = (size_t)r; return 0; @@ -113,7 +123,9 @@ static int _os_read_all(long fd, void *buf, size_t n, size_t *nread) *nread = 0; while (n>0) { + set_io_wait_begin(1); int err = _os_read(fd, buf, n, &got); + set_io_wait_begin(0); n -= got; *nread += got; buf = (char *)buf + got; @@ -844,7 +856,9 @@ static int open_cloexec(const char *path, int flags, mode_t mode) static int no_cloexec=0; if (!no_cloexec) { + set_io_wait_begin(1); int fd = open(path, flags | O_CLOEXEC, mode); + set_io_wait_begin(0); if (fd != -1) return fd; if (errno != EINVAL) @@ -852,7 +866,10 @@ static int open_cloexec(const char *path, int flags, mode_t mode) no_cloexec = 1; } #endif - return open(path, flags, mode); + set_io_wait_begin(1); + int fd = open(path, flags, mode); + set_io_wait_begin(0); + return fd; } #endif @@ -871,7 +888,9 @@ ios_t *ios_file(ios_t *s, const char *fname, int rd, int wr, int create, int tru if (!wlen) goto open_file_err; wchar_t *fname_w = (wchar_t*)alloca(wlen*sizeof(wchar_t)); if (!MultiByteToWideChar(CP_UTF8, 0, fname, -1, fname_w, wlen)) goto open_file_err; + set_io_wait_begin(1); fd = _wopen(fname_w, flags | O_BINARY | O_NOINHERIT, _S_IREAD | _S_IWRITE); + set_io_wait_begin(0); #else fd = open_cloexec(fname, flags, S_IRUSR | S_IWUSR /* 0600 */ | S_IRGRP | S_IROTH /* 0644 */); #endif diff --git a/src/support/ios.h b/src/support/ios.h index 70e9353f62bda..7f86dcac4133e 100644 --- a/src/support/ios.h +++ b/src/support/ios.h @@ -70,6 +70,7 @@ typedef struct { char local[IOS_INLSIZE]; } ios_t; +extern void (*ios_set_io_wait_func)(int); /* low-level interface functions */ JL_DLLEXPORT size_t ios_read(ios_t *s, char *dest, size_t n); JL_DLLEXPORT size_t ios_readall(ios_t *s, char *dest, size_t n); diff --git a/src/task.c b/src/task.c index 077245a302235..abd2968f88a42 100644 --- a/src/task.c +++ b/src/task.c @@ -246,6 +246,10 @@ static void NOINLINE JL_NORETURN start_task(void) } else { JL_TRY { + if (jl_get_ptls_states()->defer_signal) { + jl_get_ptls_states()->defer_signal = 0; + jl_sigint_safepoint(); + } res = jl_apply(&t->start, 1); } JL_CATCH { @@ -286,19 +290,6 @@ static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) { if (t == jl_current_task) return; - /* - making task switching interrupt-safe is going to be challenging. - we need JL_SIGATOMIC_BEGIN in jl_enter_handler, and then - JL_SIGATOMIC_END after every JL_TRY sigsetjmp that returns zero. - also protect jl_eh_restore_state. - then we need JL_SIGATOMIC_BEGIN at the top of this function (ctx_switch). - the JL_SIGATOMIC_END at the end of this function handles the case - of task switching with yieldto(). - then we need to handle the case of task switching via raise(). - to do that, the top of every catch block must do JL_SIGATOMIC_END - *IF AND ONLY IF* throwing the exception involved a task switch. - */ - //JL_SIGATOMIC_BEGIN(); if (!jl_setjmp(jl_current_task->ctx, 0)) { jl_bt_size = 0; // backtraces don't survive task switches, see e.g. issue #12485 #ifdef COPY_STACKS @@ -366,7 +357,6 @@ static void ctx_switch(jl_task_t *t, jl_jmp_buf *where) jl_longjmp(*where, 1); #endif } - //JL_SIGATOMIC_END(); } JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) @@ -385,6 +375,7 @@ JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) jl_error("task switch not allowed from inside gc finalizer"); if (in_pure_callback) jl_error("task switch not allowed from inside staged function"); + sig_atomic_t defer_signal = jl_get_ptls_states()->defer_signal; int8_t gc_state = jl_gc_unsafe_enter(); jl_task_arg_in_transit = arg; ctx_switch(t, &t->ctx); @@ -392,6 +383,10 @@ JL_DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg) jl_task_arg_in_transit = jl_nothing; throw_if_exception_set(jl_current_task); jl_gc_unsafe_leave(gc_state); + sig_atomic_t other_defer_signal = jl_get_ptls_states()->defer_signal; + jl_get_ptls_states()->defer_signal = defer_signal; + if (other_defer_signal && !defer_signal) + jl_sigint_safepoint(); return val; } @@ -496,6 +491,7 @@ static void init_task(jl_task_t *t, char *stack) // yield to exception handler void JL_NORETURN throw_internal(jl_value_t *e) { + jl_get_ptls_states()->io_wait = 0; if (jl_safe_restore) jl_longjmp(*jl_safe_restore, 1); jl_gc_unsafe_enter(); diff --git a/src/threading.c b/src/threading.c index a3a97b945f6b6..bfc47c8e3676f 100644 --- a/src/threading.c +++ b/src/threading.c @@ -104,6 +104,16 @@ static void ti_initthread(int16_t tid) ptls->tid = tid; ptls->pgcstack = NULL; ptls->gc_state = 0; // GC unsafe + // Conditionally initialize the safepoint address. See comment in + // `safepoint.c` + if (tid == 0) { + ptls->safepoint = (size_t*)(jl_safepoint_pages + jl_page_size); + } + else { + ptls->safepoint = (size_t*)(jl_safepoint_pages + jl_page_size * 2 + + sizeof(size_t)); + } + ptls->defer_signal = 0; ptls->current_module = NULL; void *bt_data = malloc(sizeof(uintptr_t) * (JL_MAX_BT_SIZE + 1)); if (bt_data == NULL) { diff --git a/src/typemap.c b/src/typemap.c index 09cbc0258a895..7a61edf6486d3 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -834,7 +834,7 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par *overwritten = ml->func.value; if (newvalue == NULL) // don't overwrite with guard entries return ml; - JL_SIGATOMIC_BEGIN(); + // sigatomic begin ml->sig = type; jl_gc_wb(ml, ml->sig); ml->simplesig = simpletype; @@ -846,7 +846,7 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par ml->func.value = newvalue; if (newvalue) jl_gc_wb(ml, newvalue); - JL_SIGATOMIC_END(); + // sigatomic end return ml; } if (overwritten != NULL) diff --git a/test/core.jl b/test/core.jl index c629ba96aa274..bbc2af9db4d35 100644 --- a/test/core.jl +++ b/test/core.jl @@ -2192,13 +2192,24 @@ end @unix_only begin ccall(:jl_exit_on_sigint, Void, (Cint,), 0) @test_throws InterruptException begin - #ccall(:raise, Void, (Cint,), 2) # llvm installs a custom version on Darwin that resolves to pthread_kill(pthread_self(), sig), which isn't what we want - Libc.systemsleep(0.1) ccall(:kill, Void, (Cint, Cint,), getpid(), 2) - Libc.systemsleep(0.2) # wait for SIGINT to arrive + for i in 1:10 + Libc.systemsleep(0.1) + ccall(:jl_gc_safepoint, Void, ()) # wait for SIGINT to arrive + end end ccall(:jl_exit_on_sigint, Void, (Cint,), 1) end +let + # Exception frame automatically restores sigatomic counter. + Base.sigatomic_begin() + @test_throws ErrorException begin + for i = 1:2 + Base.sigatomic_end() + end + end + Base.sigatomic_end() +end # pull request #9534 @test try; a,b,c = 1,2; catch ex; (ex::BoundsError).a === (1,2) && ex.i == 3; end