From e0cfd8cf0f122d1f9258f9b1e3362e1ef219a4b8 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 22 Apr 2025 20:52:09 +0000 Subject: [PATCH] reduce places where builtins are listed DRY code somewhat by consolidating builtin declarations to use a table or macro definition to auto-generate the required reflection metadata. --- src/builtin_proto.h | 152 +++++++------ src/builtins.c | 143 ++++-------- src/cgutils.cpp | 2 +- src/codegen.cpp | 312 +++++++++++--------------- src/common_symbols2.inc | 2 +- src/gf.c | 12 +- src/init.c | 2 - src/julia_internal.h | 10 +- src/llvm-codegen-shared.h | 3 - src/llvm-pass-helpers.cpp | 3 + src/method.c | 5 +- src/signals-mach.c | 2 - src/staticdata.c | 456 +++++++++++++++++--------------------- src/toplevel.c | 4 +- 14 files changed, 472 insertions(+), 636 deletions(-) diff --git a/src/builtin_proto.h b/src/builtin_proto.h index c82ec77414129..586d948f722c1 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -8,82 +8,88 @@ extern "C" { #endif // declarations for julia-callable builtin functions +#define JL_BUILTIN_FUNCTIONS(XX) \ + XX(_abstracttype,"_abstracttype") \ + XX(_apply_iterate,"_apply_iterate") \ + XX(_call_in_world_total,"_call_in_world_total") \ + XX(_compute_sparams,"_compute_sparams") \ + XX(_defaultctors,"_defaultctors") \ + XX(_equiv_typedef,"_equiv_typedef") \ + XX(_expr,"_expr") \ + XX(_primitivetype,"_primitivetype") \ + XX(_setsuper,"_setsuper!") \ + XX(_structtype,"_structtype") \ + XX(_svec_ref,"_svec_ref") \ + XX(_typebody,"_typebody!") \ + XX(_typevar,"_typevar") \ + XX(applicable,"applicable") \ + XX(apply_type,"apply_type") \ + XX(compilerbarrier,"compilerbarrier") \ + XX(current_scope,"current_scope") \ + XX(donotdelete,"donotdelete") \ + XX(fieldtype,"fieldtype") \ + XX(finalizer,"finalizer") \ + XX(get_binding_type,"get_binding_type") \ + XX(getfield,"getfield") \ + XX(getglobal,"getglobal") \ + XX(ifelse,"ifelse") \ + XX(intrinsic_call,"intrinsic_call") \ + XX(invoke,"invoke") \ + XX(invoke_in_world,"invoke_in_world") \ + XX(invokelatest,"invokelatest") \ + XX(is,"===") \ + XX(isa,"isa") \ + XX(isdefined,"isdefined") \ + XX(isdefinedglobal,"isdefinedglobal") \ + XX(issubtype,"<:") \ + XX(memorynew,"memorynew") \ + XX(memoryrefnew,"memoryrefnew") \ + XX(memoryref_isassigned,"memoryref_isassigned") \ + XX(memoryrefget,"memoryrefget") \ + XX(memoryrefmodify,"memoryrefmodify!") \ + XX(memoryrefoffset,"memoryrefoffset") \ + XX(memoryrefreplace,"memoryrefreplace!") \ + XX(memoryrefset,"memoryrefset!") \ + XX(memoryrefsetonce,"memoryrefsetonce!") \ + XX(memoryrefswap,"memoryrefswap!") \ + XX(modifyfield,"modifyfield!") \ + XX(modifyglobal,"modifyglobal!") \ + XX(nfields,"nfields") \ + XX(opaque_closure_call,"opaque_closure_call") \ + XX(replacefield,"replacefield!") \ + XX(replaceglobal,"replaceglobal!") \ + XX(setfield,"setfield!") \ + XX(setfieldonce,"setfieldonce!") \ + XX(setglobal,"setglobal!") \ + XX(setglobalonce,"setglobalonce!") \ + XX(sizeof,"sizeof") \ + XX(svec,"svec") \ + XX(swapfield,"swapfield!") \ + XX(swapglobal,"swapglobal!") \ + XX(throw,"throw") \ + XX(throw_methoderror,"throw_methoderror") \ + XX(tuple,"tuple") \ + XX(typeassert,"typeassert") \ + XX(typeof,"typeof") \ -#ifdef DEFINE_BUILTIN_GLOBALS -#define DECLARE_BUILTIN(name) \ - JL_CALLABLE(jl_f_##name); \ - JL_DLLEXPORT jl_value_t *jl_builtin_##name; \ - JL_DLLEXPORT jl_fptr_args_t jl_f_##name##_addr = &jl_f_##name -#else -#define DECLARE_BUILTIN(name) \ - JL_CALLABLE(jl_f_##name); \ - JL_DLLEXPORT extern jl_value_t *jl_builtin_##name; \ - JL_DLLEXPORT extern jl_fptr_args_t jl_f_##name##_addr -#endif +#define DECLARE_BUILTIN(cname,jlname) \ + JL_CALLABLE(jl_f_##cname); +JL_BUILTIN_FUNCTIONS(DECLARE_BUILTIN) +#undef DECLARE_BUILTIN + +#define BUILTIN(cname) (jl_builtin_instances[jl_builtin_id_##cname]) + +enum jl_builtin_ids { +#define BUILTIN_IDS(cname,jlname) jl_builtin_id_##cname, +JL_BUILTIN_FUNCTIONS(BUILTIN_IDS) +#undef BUILTIN_IDS + jl_n_builtins +}; -DECLARE_BUILTIN(_apply_iterate); -DECLARE_BUILTIN(invoke_in_world); -DECLARE_BUILTIN(_call_in_world_total); -DECLARE_BUILTIN(invokelatest); -DECLARE_BUILTIN(_compute_sparams); -DECLARE_BUILTIN(_expr); -DECLARE_BUILTIN(_svec_ref); -DECLARE_BUILTIN(_typebody); -DECLARE_BUILTIN(_typevar); -DECLARE_BUILTIN(applicable); -DECLARE_BUILTIN(apply_type); -DECLARE_BUILTIN(compilerbarrier); -DECLARE_BUILTIN(current_scope); -DECLARE_BUILTIN(donotdelete); -DECLARE_BUILTIN(fieldtype); -DECLARE_BUILTIN(finalizer); -DECLARE_BUILTIN(getfield); -DECLARE_BUILTIN(getglobal); -DECLARE_BUILTIN(ifelse); -DECLARE_BUILTIN(invoke); -DECLARE_BUILTIN(is); -DECLARE_BUILTIN(isa); -DECLARE_BUILTIN(isdefined); -DECLARE_BUILTIN(isdefinedglobal); -DECLARE_BUILTIN(issubtype); -DECLARE_BUILTIN(memorynew); -DECLARE_BUILTIN(memoryref); -DECLARE_BUILTIN(memoryref_isassigned); -DECLARE_BUILTIN(memoryrefget); -DECLARE_BUILTIN(memoryrefmodify); -DECLARE_BUILTIN(memoryrefoffset); -DECLARE_BUILTIN(memoryrefreplace); -DECLARE_BUILTIN(memoryrefset); -DECLARE_BUILTIN(memoryrefsetonce); -DECLARE_BUILTIN(memoryrefswap); -DECLARE_BUILTIN(modifyfield); -DECLARE_BUILTIN(modifyglobal); -DECLARE_BUILTIN(nfields); -DECLARE_BUILTIN(replacefield); -DECLARE_BUILTIN(replaceglobal); -DECLARE_BUILTIN(setfield); -DECLARE_BUILTIN(setfieldonce); -DECLARE_BUILTIN(setglobal); -DECLARE_BUILTIN(setglobalonce); -DECLARE_BUILTIN(sizeof); -DECLARE_BUILTIN(svec); -DECLARE_BUILTIN(swapfield); -DECLARE_BUILTIN(swapglobal); -DECLARE_BUILTIN(throw); -DECLARE_BUILTIN(throw_methoderror); -DECLARE_BUILTIN(tuple); -DECLARE_BUILTIN(typeassert); -DECLARE_BUILTIN(typeof); +JL_DLLEXPORT extern jl_fptr_args_t const jl_builtin_f_addrs[]; +JL_DLLEXPORT extern const char *const jl_builtin_f_names[]; +JL_DLLEXPORT extern jl_value_t *jl_builtin_instances[]; -JL_CALLABLE(jl_f__structtype); -JL_CALLABLE(jl_f__abstracttype); -JL_CALLABLE(jl_f__primitivetype); -JL_CALLABLE(jl_f__setsuper); -JL_CALLABLE(jl_f__defaultctors); -JL_CALLABLE(jl_f__equiv_typedef); -JL_CALLABLE(jl_f_get_binding_type); -JL_CALLABLE(jl_f__compute_sparams); -JL_CALLABLE(jl_f__svec_ref); #ifdef __cplusplus } #endif diff --git a/src/builtins.c b/src/builtins.c index a2cae857f26b4..e3a0380182e15 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -30,6 +30,27 @@ extern "C" { #endif +jl_fptr_args_t const jl_builtin_f_addrs[jl_n_builtins] = { +#define BUILTIN_ADDRS(cname,jlname) &jl_f_##cname, +JL_BUILTIN_FUNCTIONS(BUILTIN_ADDRS) +#undef BUILTIN_ADDRS +}; + +const char *const jl_builtin_f_names[jl_n_builtins] = { +#define BUILTIN_F_NAMES(cname,jlname) XSTR(jl_f_##cname), +JL_BUILTIN_FUNCTIONS(BUILTIN_F_NAMES) +#undef BUILTIN_F_NAMES +}; + +jl_value_t *jl_builtin_instances[jl_n_builtins]; + +static const char *const jl_builtin_names[jl_n_builtins] = { +#define BUILTIN_NAMES(cname,jlname) jlname, +JL_BUILTIN_FUNCTIONS(BUILTIN_NAMES) +#undef BUILTIN_NAMES +}; + + // egal and object_id --------------------------------------------------------- static int bits_equal(const void *a, const void *b, int sz) JL_NOTSAFEPOINT @@ -647,7 +668,7 @@ JL_CALLABLE(jl_f__apply_iterate) nargs -= 1; if (nargs == 2) { // some common simple cases - if (f == jl_builtin_svec) { + if (f == BUILTIN(svec)) { if (jl_is_svec(args[1])) return args[1]; if (jl_is_genericmemory(args[1])) { @@ -672,7 +693,7 @@ JL_CALLABLE(jl_f__apply_iterate) return (jl_value_t*)t; } } - else if (f == jl_builtin_tuple && jl_is_tuple(args[1])) { + else if (f == BUILTIN(tuple) && jl_is_tuple(args[1])) { return args[1]; } } @@ -1691,11 +1712,11 @@ JL_CALLABLE(jl_f_memorynew) return (jl_value_t*)jl_alloc_genericmemory(args[0], nel); } -JL_CALLABLE(jl_f_memoryref) +JL_CALLABLE(jl_f_memoryrefnew) { - JL_NARGS(memoryref, 1, 3); + JL_NARGS(memoryrefnew, 1, 3); if (nargs == 1) { - JL_TYPECHK(memoryref, genericmemory, args[0]); + JL_TYPECHK(memoryrefnew, genericmemory, args[0]); jl_genericmemory_t *m = (jl_genericmemory_t*)args[0]; jl_value_t *typ = jl_apply_type((jl_value_t*)jl_genericmemoryref_type, jl_svec_data(((jl_datatype_t*)jl_typetagof(m))->parameters), 3); JL_GC_PROMISE_ROOTED(typ); // it is a concrete type @@ -1705,10 +1726,10 @@ JL_CALLABLE(jl_f_memoryref) return (jl_value_t*)jl_new_memoryref(typ, m, m->ptr); } else { - JL_TYPECHK(memoryref, genericmemoryref, args[0]); - JL_TYPECHK(memoryref, long, args[1]); + JL_TYPECHK(memoryrefnew, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefnew, long, args[1]); if (nargs == 3) - JL_TYPECHK(memoryref, bool, args[2]); + JL_TYPECHK(memoryrefnew, bool, args[2]); jl_genericmemoryref_t *m = (jl_genericmemoryref_t*)args[0]; size_t i = jl_unbox_long(args[1]) - 1; const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m->mem))->layout; @@ -1735,7 +1756,7 @@ JL_CALLABLE(jl_f_memoryref) JL_CALLABLE(jl_f_memoryrefoffset) { JL_NARGS(memoryrefoffset, 1, 1); - JL_TYPECHK(memoryref, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefoffest, genericmemoryref, args[0]); jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; size_t offset; @@ -2415,10 +2436,10 @@ void jl_init_intrinsic_functions(void) JL_GC_DISABLED { jl_module_t *inm = jl_new_module_(jl_symbol("Intrinsics"), jl_core_module, 0, 1); jl_set_initial_const(jl_core_module, jl_symbol("Intrinsics"), (jl_value_t*)inm, 0); - jl_mk_builtin_func(jl_intrinsic_type, "IntrinsicFunction", jl_f_intrinsic_call); + jl_mk_builtin_func(jl_intrinsic_type, jl_symbol("IntrinsicFunction"), jl_f_intrinsic_call); jl_mk_builtin_func( (jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_opaque_closure_type), - "OpaqueClosure", jl_f_opaque_closure_call); + jl_symbol("OpaqueClosure"), jl_f_opaque_closure_call); // Save a reference to the just created OpaqueClosure method, so we can provide special // codegen for it later. @@ -2439,93 +2460,21 @@ static void add_builtin(const char *name, jl_value_t *v) jl_set_initial_const(jl_core_module, jl_symbol(name), v, 0); } -jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt) -{ - assert(jl_subtype((jl_value_t*)dt, (jl_value_t*)jl_builtin_type)); - jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_atomic_load_relaxed(&dt->name->mt->defs); - jl_method_instance_t *mi = jl_atomic_load_relaxed(&entry->func.method->unspecialized); - jl_code_instance_t *ci = jl_atomic_load_relaxed(&mi->cache); - assert(ci->owner == jl_nothing); - return jl_atomic_load_relaxed(&ci->specptr.fptr1); -} - -static jl_value_t *add_builtin_func(const char *name, jl_fptr_args_t fptr) -{ - return jl_mk_builtin_func(NULL, name, fptr)->instance; -} - void jl_init_primitives(void) JL_GC_DISABLED { - jl_builtin_is = add_builtin_func("===", jl_f_is); - jl_builtin_typeof = add_builtin_func("typeof", jl_f_typeof); - jl_builtin_sizeof = add_builtin_func("sizeof", jl_f_sizeof); - jl_builtin_issubtype = add_builtin_func("<:", jl_f_issubtype); - jl_builtin_isa = add_builtin_func("isa", jl_f_isa); - jl_builtin_typeassert = add_builtin_func("typeassert", jl_f_typeassert); - jl_builtin_throw = add_builtin_func("throw", jl_f_throw); - jl_builtin_tuple = add_builtin_func("tuple", jl_f_tuple); - jl_builtin_ifelse = add_builtin_func("ifelse", jl_f_ifelse); - - // field access - jl_builtin_getfield = add_builtin_func("getfield", jl_f_getfield); - jl_builtin_setfield = add_builtin_func("setfield!", jl_f_setfield); - jl_builtin_setfieldonce = add_builtin_func("setfieldonce!", jl_f_setfieldonce); - jl_builtin_swapfield = add_builtin_func("swapfield!", jl_f_swapfield); - jl_builtin_modifyfield = add_builtin_func("modifyfield!", jl_f_modifyfield); - jl_builtin_replacefield = add_builtin_func("replacefield!", jl_f_replacefield); - jl_builtin_fieldtype = add_builtin_func("fieldtype", jl_f_fieldtype); - jl_builtin_nfields = add_builtin_func("nfields", jl_f_nfields); - jl_builtin_isdefined = add_builtin_func("isdefined", jl_f_isdefined); - - // module bindings - jl_builtin_getglobal = add_builtin_func("getglobal", jl_f_getglobal); - jl_builtin_setglobal = add_builtin_func("setglobal!", jl_f_setglobal); - jl_builtin_isdefinedglobal = add_builtin_func("isdefinedglobal", jl_f_isdefinedglobal); - add_builtin_func("get_binding_type", jl_f_get_binding_type); - jl_builtin_swapglobal = add_builtin_func("swapglobal!", jl_f_swapglobal); - jl_builtin_replaceglobal = add_builtin_func("replaceglobal!", jl_f_replaceglobal); - jl_builtin_modifyglobal = add_builtin_func("modifyglobal!", jl_f_modifyglobal); - jl_builtin_setglobalonce = add_builtin_func("setglobalonce!", jl_f_setglobalonce); - - // memory primitives - jl_builtin_memorynew = add_builtin_func("memorynew", jl_f_memorynew); - jl_builtin_memoryref = add_builtin_func("memoryrefnew", jl_f_memoryref); - jl_builtin_memoryrefoffset = add_builtin_func("memoryrefoffset", jl_f_memoryrefoffset); - jl_builtin_memoryrefget = add_builtin_func("memoryrefget", jl_f_memoryrefget); - jl_builtin_memoryrefset = add_builtin_func("memoryrefset!", jl_f_memoryrefset); - jl_builtin_memoryref_isassigned = add_builtin_func("memoryref_isassigned", jl_f_memoryref_isassigned); - jl_builtin_memoryrefswap = add_builtin_func("memoryrefswap!", jl_f_memoryrefswap); - jl_builtin_memoryrefreplace = add_builtin_func("memoryrefreplace!", jl_f_memoryrefreplace); - jl_builtin_memoryrefmodify = add_builtin_func("memoryrefmodify!", jl_f_memoryrefmodify); - jl_builtin_memoryrefsetonce = add_builtin_func("memoryrefsetonce!", jl_f_memoryrefsetonce); - - // method table utils - jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable); - jl_builtin_invoke = add_builtin_func("invoke", jl_f_invoke); - - // internal functions - jl_builtin_apply_type = add_builtin_func("apply_type", jl_f_apply_type); - jl_builtin__apply_iterate = add_builtin_func("_apply_iterate", jl_f__apply_iterate); - jl_builtin__expr = add_builtin_func("_expr", jl_f__expr); - jl_builtin_svec = add_builtin_func("svec", jl_f_svec); - add_builtin_func("invokelatest", jl_f_invokelatest); - add_builtin_func("invoke_in_world", jl_f_invoke_in_world); - add_builtin_func("_call_in_world_total", jl_f__call_in_world_total); - add_builtin_func("_typevar", jl_f__typevar); - add_builtin_func("_structtype", jl_f__structtype); - add_builtin_func("_abstracttype", jl_f__abstracttype); - add_builtin_func("_primitivetype", jl_f__primitivetype); - add_builtin_func("_setsuper!", jl_f__setsuper); - add_builtin_func("_defaultctors", jl_f__defaultctors); - jl_builtin__typebody = add_builtin_func("_typebody!", jl_f__typebody); - add_builtin_func("_equiv_typedef", jl_f__equiv_typedef); - jl_builtin_donotdelete = add_builtin_func("donotdelete", jl_f_donotdelete); - jl_builtin_compilerbarrier = add_builtin_func("compilerbarrier", jl_f_compilerbarrier); - add_builtin_func("finalizer", jl_f_finalizer); - add_builtin_func("_compute_sparams", jl_f__compute_sparams); - add_builtin_func("_svec_ref", jl_f__svec_ref); - jl_builtin_current_scope = add_builtin_func("current_scope", jl_f_current_scope); - add_builtin_func("throw_methoderror", jl_f_throw_methoderror); + // Builtins are specially considered available from world 0 + for (int i = 0; i < jl_n_builtins; i++) { + if (i == jl_builtin_id_intrinsic_call || + i == jl_builtin_id_opaque_closure_call) + continue; + jl_sym_t *sname = jl_symbol(jl_builtin_names[i]); + jl_value_t *builtin = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); + jl_set_initial_const(jl_core_module, sname, builtin, 0); + jl_mk_builtin_func((jl_datatype_t*)jl_typeof(builtin), sname, jl_builtin_f_addrs[i]); + jl_builtin_instances[i] = builtin; + } + add_builtin("OpaqueClosure", (jl_value_t*)jl_opaque_closure_type); + add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type); // builtin types add_builtin("Any", (jl_value_t*)jl_any_type); @@ -2558,14 +2507,12 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("PartialOpaque", (jl_value_t*)jl_partial_opaque_type); add_builtin("InterConditional", (jl_value_t*)jl_interconditional_type); add_builtin("MethodMatch", (jl_value_t*)jl_method_match_type); - add_builtin("IntrinsicFunction", (jl_value_t*)jl_intrinsic_type); add_builtin("Function", (jl_value_t*)jl_function_type); add_builtin("Builtin", (jl_value_t*)jl_builtin_type); add_builtin("MethodInstance", (jl_value_t*)jl_method_instance_type); add_builtin("CodeInfo", (jl_value_t*)jl_code_info_type); add_builtin("LLVMPtr", (jl_value_t*)jl_llvmpointer_type); add_builtin("Task", (jl_value_t*)jl_task_type); - add_builtin("OpaqueClosure", (jl_value_t*)jl_opaque_closure_type); add_builtin("AddrSpace", (jl_value_t*)jl_addrspace_type); add_builtin("Ref", (jl_value_t*)jl_ref_type); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index d9b7b98e40ef4..cfa40b130eb97 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4662,7 +4662,7 @@ static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, const static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cgval_t idx, jl_value_t *inbounds, const jl_datatype_layout_t *layout) { ++EmittedArrayNdIndex; - emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryref"); + emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryrefnew"); idx = update_julia_type(ctx, idx, (jl_value_t*)jl_long_type); if (idx.typ == jl_bottom_type) return jl_cgval_t(); diff --git a/src/codegen.cpp b/src/codegen.cpp index c89fc423bd948..a4817aa0e9bc8 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -341,7 +341,7 @@ struct jl_tbaacache_t { MDNode *tbaa_ptrarraybuf; // Data in an array of boxed values MDNode *tbaa_arraybuf; // Data in an array of POD MDNode *tbaa_array; // jl_array_t or jl_genericmemory_t - MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t (to memoryref) + MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t (to a memoryref) MDNode *tbaa_arraysize; // A size in a jl_array_t MDNode *tbaa_arrayselbyte; // a selector byte in a isbits Union jl_genericmemory_t MDNode *tbaa_memoryptr; // The pointer inside a jl_genericmemory_t @@ -555,7 +555,14 @@ FunctionType *invoke_type(TypeFnContextAndTriple f, Module &M) template struct JuliaFunction { public: - llvm::StringLiteral name; + template + constexpr JuliaFunction(const char (&cname)[N], TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) + : name(StringRef(cname, N-1)), _type(_type), _attrs(_attrs) {} + JuliaFunction(StringRef cname, TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) + : name(cname), _type(_type), _attrs(_attrs) {} + JuliaFunction(char *cname, TypeFn_t _type, llvm::AttributeList (*_attrs)(llvm::LLVMContext &C)) = delete; + + llvm::StringRef name; TypeFn_t _type; llvm::AttributeList (*_attrs)(llvm::LLVMContext &C); @@ -604,10 +611,6 @@ static FunctionType *get_func_sig(LLVMContext &C) { return JuliaType::get_jlfunc static FunctionType *get_func2_sig(LLVMContext &C) { return JuliaType::get_jlfunc2_ty(C); } static FunctionType *get_func3_sig(LLVMContext &C) { return JuliaType::get_jlfunc3_ty(C); } -static FunctionType *get_donotdelete_sig(LLVMContext &C) { - return FunctionType::get(getVoidTy(C), true); -} - static AttributeList get_func_attrs(LLVMContext &C) { return AttributeList::get(C, @@ -617,18 +620,6 @@ static AttributeList get_func_attrs(LLVMContext &C) Attributes(C, {Attribute::NoAlias, Attribute::ReadOnly, Attribute::NoCapture, Attribute::NoUndef})}); } -static AttributeList get_donotdelete_func_attrs(LLVMContext &C) -{ - AttrBuilder FnAttrs(C); - FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); - FnAttrs.addAttribute(Attribute::WillReturn); - FnAttrs.addAttribute(Attribute::NoUnwind); - return AttributeList::get(C, - AttributeSet::get(C, FnAttrs), - Attributes(C, {}), - None); -} - static AttributeList get_attrs_noreturn(LLVMContext &C) { return AttributeList::get(C, @@ -1440,11 +1431,21 @@ static const auto box_ssavalue_func = new JuliaFunction{ }, get_attrs_basic, }; -static const auto jlgetbuiltinfptr_func = new JuliaFunction<>{ - XSTR(jl_get_builtin_fptr), - [](LLVMContext &C) { return FunctionType::get(getPointerTy(C), - {JuliaType::get_prjlvalue_ty(C)}, false); }, - nullptr, +static const auto jldnd_func = new JuliaFunction<>{ + XSTR(jl_f_donotdelete), + [](LLVMContext &C) { + return FunctionType::get(getVoidTy(C), true); + }, + [](LLVMContext &C) { + AttrBuilder FnAttrs(C); + FnAttrs.addMemoryAttr(MemoryEffects::inaccessibleMemOnly()); + FnAttrs.addAttribute(Attribute::WillReturn); + FnAttrs.addAttribute(Attribute::NoUnwind); + return AttributeList::get(C, + AttributeSet::get(C, FnAttrs), + Attributes(C, {}), + None); + }, }; // placeholder functions @@ -1558,56 +1559,23 @@ static const auto julia_call3 = new JuliaFunction<>{ static const auto jltuple_func = new JuliaFunction<>{XSTR(jl_f_tuple), get_func_sig, get_func_attrs}; static const auto jlintrinsic_func = new JuliaFunction<>{XSTR(jl_f_intrinsic_call), get_func3_sig, get_func_attrs}; +static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; + +static const auto mk_builtin_func_map() { + auto builtin_addrs = new DenseMap*>(); + for (int i = 0; i < jl_n_builtins; i++) { + jl_value_t *builtin = jl_builtin_instances[i]; + if (builtin) // a couple do not have instances (e.g. IntrinsicFunction) + (*builtin_addrs)[builtin] = new JuliaFunction<>{StringRef(jl_builtin_f_names[i]), get_func_sig, get_func_attrs}; + } + return builtin_addrs; +} static const auto &builtin_func_map() { - static auto builtins = new DenseMap*> { - { jl_f_is_addr, new JuliaFunction<>{XSTR(jl_f_is), get_func_sig, get_func_attrs} }, - { jl_f_typeof_addr, new JuliaFunction<>{XSTR(jl_f_typeof), get_func_sig, get_func_attrs} }, - { jl_f_sizeof_addr, new JuliaFunction<>{XSTR(jl_f_sizeof), get_func_sig, get_func_attrs} }, - { jl_f_issubtype_addr, new JuliaFunction<>{XSTR(jl_f_issubtype), get_func_sig, get_func_attrs} }, - { jl_f_isa_addr, new JuliaFunction<>{XSTR(jl_f_isa), get_func_sig, get_func_attrs} }, - { jl_f_typeassert_addr, new JuliaFunction<>{XSTR(jl_f_typeassert), get_func_sig, get_func_attrs} }, - { jl_f_ifelse_addr, new JuliaFunction<>{XSTR(jl_f_ifelse), get_func_sig, get_func_attrs} }, - { jl_f__apply_iterate_addr, new JuliaFunction<>{XSTR(jl_f__apply_iterate), get_func_sig, get_func_attrs} }, - { jl_f_invokelatest_addr, new JuliaFunction<>{XSTR(jl_f_invokelatest), get_func_sig, get_func_attrs} }, - { jl_f_invoke_in_world_addr, new JuliaFunction<>{XSTR(jl_f_invoke_in_world), get_func_sig, get_func_attrs} }, - { jl_f__call_in_world_total_addr, new JuliaFunction<>{XSTR(jl_f__call_in_world_total), get_func_sig, get_func_attrs} }, - { jl_f_throw_addr, new JuliaFunction<>{XSTR(jl_f_throw), get_func_sig, get_func_attrs} }, - { jl_f_throw_methoderror_addr, new JuliaFunction<>{XSTR(jl_f_throw_methoderror), get_func_sig, get_func_attrs} }, - { jl_f_tuple_addr, jltuple_func }, - { jl_f_svec_addr, new JuliaFunction<>{XSTR(jl_f_svec), get_func_sig, get_func_attrs} }, - { jl_f_applicable_addr, new JuliaFunction<>{XSTR(jl_f_applicable), get_func_sig, get_func_attrs} }, - { jl_f_invoke_addr, new JuliaFunction<>{XSTR(jl_f_invoke), get_func_sig, get_func_attrs} }, - { jl_f_isdefined_addr, new JuliaFunction<>{XSTR(jl_f_isdefined), get_func_sig, get_func_attrs} }, - { jl_f_getfield_addr, new JuliaFunction<>{XSTR(jl_f_getfield), get_func_sig, get_func_attrs} }, - { jl_f_setfield_addr, new JuliaFunction<>{XSTR(jl_f_setfield), get_func_sig, get_func_attrs} }, - { jl_f_swapfield_addr, new JuliaFunction<>{XSTR(jl_f_swapfield), get_func_sig, get_func_attrs} }, - { jl_f_modifyfield_addr, new JuliaFunction<>{XSTR(jl_f_modifyfield), get_func_sig, get_func_attrs} }, - { jl_f_fieldtype_addr, new JuliaFunction<>{XSTR(jl_f_fieldtype), get_func_sig, get_func_attrs} }, - { jl_f_nfields_addr, new JuliaFunction<>{XSTR(jl_f_nfields), get_func_sig, get_func_attrs} }, - { jl_f__expr_addr, new JuliaFunction<>{XSTR(jl_f__expr), get_func_sig, get_func_attrs} }, - { jl_f__typevar_addr, new JuliaFunction<>{XSTR(jl_f__typevar), get_func_sig, get_func_attrs} }, - { jl_f_memorynew_addr, new JuliaFunction<>{XSTR(jl_f_memorynew), get_func_sig, get_func_attrs} }, - { jl_f_memoryref_addr, new JuliaFunction<>{XSTR(jl_f_memoryref), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefoffset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefoffset), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefset), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefswap_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefswap), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefreplace_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefreplace), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefmodify_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefmodify), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefsetonce_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefsetonce), get_func_sig, get_func_attrs} }, - { jl_f_memoryref_isassigned_addr,new JuliaFunction<>{XSTR(jl_f_memoryref_isassigned), get_func_sig, get_func_attrs} }, - { jl_f_apply_type_addr, new JuliaFunction<>{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} }, - { jl_f_donotdelete_addr, new JuliaFunction<>{XSTR(jl_f_donotdelete), get_donotdelete_sig, get_donotdelete_func_attrs} }, - { jl_f_compilerbarrier_addr, new JuliaFunction<>{XSTR(jl_f_compilerbarrier), get_func_sig, get_func_attrs} }, - { jl_f_finalizer_addr, new JuliaFunction<>{XSTR(jl_f_finalizer), get_func_sig, get_func_attrs} }, - { jl_f__svec_ref_addr, new JuliaFunction<>{XSTR(jl_f__svec_ref), get_func_sig, get_func_attrs} }, - { jl_f_current_scope_addr, new JuliaFunction<>{XSTR(jl_f_current_scope), get_func_sig, get_func_attrs} }, - }; + static auto builtins = mk_builtin_func_map(); return *builtins; } -static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; - static _Atomic(uint64_t) globalUniqueGeneratedNames{1}; // --- code generation --- @@ -2901,7 +2869,7 @@ static jl_value_t *static_apply_type(jl_codectx_t &ctx, ArrayRef arg return NULL; v[i] = args[i].constant; } - assert(v[0] == jl_builtin_apply_type); + assert(v[0] == BUILTIN(apply_type)); size_t last_age = jl_current_task->world_age; // call apply_type, but ignore errors. we know that will work in world 1. jl_current_task->world_age = 1; @@ -2971,7 +2939,7 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) if (e->head == jl_call_sym) { jl_value_t *f = static_eval(ctx, jl_exprarg(e, 0)); if (f) { - if (jl_array_dim0(e->args) == 3 && (f == jl_builtin_getfield || f == jl_builtin_getglobal)) { + if (jl_array_dim0(e->args) == 3 && (f == BUILTIN(getfield) || f == BUILTIN(getglobal))) { m = (jl_module_t*)static_eval(ctx, jl_exprarg(e, 1)); // Check the tag before evaluating `s` so that a value of random // type won't be corrupted. @@ -2990,10 +2958,11 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) } } } - else if (f==jl_builtin_tuple || f==jl_builtin_apply_type) { + else if (f==BUILTIN(tuple) || f==BUILTIN(apply_type)) { size_t i; size_t n = jl_array_dim0(e->args)-1; - if (n==0 && f==jl_builtin_tuple) return (jl_value_t*)jl_emptytuple; + if (n==0 && f==BUILTIN(tuple)) + return (jl_value_t*)jl_emptytuple; jl_value_t **v; JL_GC_PUSHARGS(v, n+1); v[0] = f; @@ -3625,11 +3594,11 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva static bool emit_f_opglobal(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { - bool issetglobal = f == jl_builtin_setglobal; - bool isreplaceglobal = f == jl_builtin_replaceglobal; - bool isswapglobal = f == jl_builtin_swapglobal; - bool ismodifyglobal = f == jl_builtin_modifyglobal; - bool issetglobalonce = f == jl_builtin_setglobalonce; + bool issetglobal = f == BUILTIN(setglobal); + bool isreplaceglobal = f == BUILTIN(replaceglobal); + bool isswapglobal = f == BUILTIN(swapglobal); + bool ismodifyglobal = f == BUILTIN(modifyglobal); + bool issetglobalonce = f == BUILTIN(setglobalonce); const jl_cgval_t undefval; const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; @@ -3698,11 +3667,11 @@ static bool emit_f_opfield(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { ++EmittedOpfields; - bool issetfield = f == jl_builtin_setfield; - bool isreplacefield = f == jl_builtin_replacefield; - bool isswapfield = f == jl_builtin_swapfield; - bool ismodifyfield = f == jl_builtin_modifyfield; - bool issetfieldonce = f == jl_builtin_setfieldonce; + bool issetfield = f == BUILTIN(setfield); + bool isreplacefield = f == BUILTIN(replacefield); + bool isswapfield = f == BUILTIN(swapfield); + bool ismodifyfield = f == BUILTIN(modifyfield); + bool issetfieldonce = f == BUILTIN(setfieldonce); const jl_cgval_t undefval; const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; @@ -3845,11 +3814,11 @@ static jl_cgval_t emit_isdefinedglobal(jl_codectx_t &ctx, jl_module_t *modu, jl_ static bool emit_f_opmemory(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ArrayRef argv, size_t nargs, const jl_cgval_t *modifyop) { - bool issetmemory = f == jl_builtin_memoryrefset; - bool isreplacememory = f == jl_builtin_memoryrefreplace; - bool isswapmemory = f == jl_builtin_memoryrefswap; - bool ismodifymemory = f == jl_builtin_memoryrefmodify; - bool issetmemoryonce = f == jl_builtin_memoryrefsetonce; + bool issetmemory = f == BUILTIN(memoryrefset); + bool isreplacememory = f == BUILTIN(memoryrefreplace); + bool isswapmemory = f == BUILTIN(memoryrefswap); + bool ismodifymemory = f == BUILTIN(memoryrefmodify); + bool issetmemoryonce = f == BUILTIN(memoryrefsetonce); const jl_cgval_t undefval; const jl_cgval_t &ref = argv[1]; @@ -4035,14 +4004,19 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // returns true if the call has been handled { ++EmittedBuiltinCalls; - if (f == jl_builtin_is && nargs == 2) { + if (f == BUILTIN(is) && nargs == 2) { // emit comparison test Value *ans = emit_f_is(ctx, argv[1], argv[2]); *ret = mark_julia_type(ctx, ans, false, jl_bool_type); return true; } - else if (f == jl_builtin_typeof && nargs == 1) { + else if (f == BUILTIN(ifelse) && nargs == 3) { + *ret = emit_ifelse(ctx, argv[1], argv[2], argv[3], rt); + return true; + } + + else if (f == BUILTIN(typeof) && nargs == 1) { const jl_cgval_t &p = argv[1]; if (p.constant) *ret = mark_julia_const(ctx, jl_typeof(p.constant)); @@ -4053,7 +4027,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_typeassert && nargs == 2) { + else if (f == BUILTIN(typeassert) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; if (jl_is_type_type(ty.typ) && !jl_has_free_typevars(ty.typ)) { @@ -4071,7 +4045,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_isa && nargs == 2) { + else if (f == BUILTIN(isa) && nargs == 2) { const jl_cgval_t &arg = argv[1]; const jl_cgval_t &ty = argv[2]; if (jl_is_type_type(ty.typ) && !jl_has_free_typevars(ty.typ)) { @@ -4082,7 +4056,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_issubtype && nargs == 2) { + else if (f == BUILTIN(issubtype) && nargs == 2) { const jl_cgval_t &ta = argv[1]; const jl_cgval_t &tb = argv[2]; if (jl_is_type_type(ta.typ) && !jl_has_free_typevars(ta.typ) && @@ -4093,7 +4067,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if ((f == jl_builtin__apply_iterate && nargs == 3) && ctx.vaSlot > 0) { + else if ((f == BUILTIN(_apply_iterate) && nargs == 3) && ctx.vaSlot > 0) { // turn Core._apply_iterate(iter, f, Tuple) ==> f(Tuple...) using the jlcall calling convention if Tuple is the va allocation if (LoadInst *load = dyn_cast_or_null(argv[3].V)) { if (load->getPointerOperand() == ctx.slots[ctx.vaSlot].boxroot && ctx.argArray) { @@ -4110,7 +4084,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_tuple) { + else if (f == BUILTIN(tuple)) { if (nargs == 0) { *ret = ghostValue(ctx, jl_emptytuple_type); return true; @@ -4121,14 +4095,14 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_throw && nargs == 1) { + else if (f == BUILTIN(throw) && nargs == 1) { Value *arg1 = boxed(ctx, argv[1]); raise_exception(ctx, arg1); *ret = jl_cgval_t(); return true; } - else if (f == jl_builtin_memorynew && (nargs == 2)) { + else if (f == BUILTIN(memorynew) && (nargs == 2)) { const jl_cgval_t &memty = argv[1]; if (!memty.constant) return false; @@ -4152,7 +4126,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_memoryref && nargs == 1) { + else if (f == BUILTIN(memoryrefnew) && nargs == 1) { const jl_cgval_t &mem = argv[1]; jl_datatype_t *mty_dt = (jl_datatype_t*)jl_unwrap_unionall(mem.typ); if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type((jl_value_t*)mty_dt)) { @@ -4163,7 +4137,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_memoryref && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(memoryrefnew) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4171,13 +4145,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; if (nargs == 3) - emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefnew"); *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout); return true; } } - else if (f == jl_builtin_memoryrefoffset && nargs == 1) { + else if (f == BUILTIN(memoryrefoffset) && nargs == 1) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4188,7 +4162,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_memoryrefget && nargs == 3) { + else if (f == BUILTIN(memoryrefget) && nargs == 3) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4227,7 +4201,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, order = isatomic ? jl_memory_order_unordered : jl_memory_order_notatomic; } jl_value_t *boundscheck = argv[3].constant; - emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryrefget"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); @@ -4304,16 +4278,16 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if ((f == jl_builtin_memoryrefset && nargs == 4) || - (f == jl_builtin_memoryrefswap && nargs == 4) || - (f == jl_builtin_memoryrefreplace && nargs == 6) || - (f == jl_builtin_memoryrefmodify && nargs == 5) || - (f == jl_builtin_memoryrefsetonce && nargs == 5)) { + else if ((f == BUILTIN(memoryrefset) && nargs == 4) || + (f == BUILTIN(memoryrefswap) && nargs == 4) || + (f == BUILTIN(memoryrefreplace) && nargs == 6) || + (f == BUILTIN(memoryrefmodify) && nargs == 5) || + (f == BUILTIN(memoryrefsetonce) && nargs == 5)) { return emit_f_opmemory(ctx, ret, f, argv, nargs, nullptr); } - else if (f == jl_builtin_memoryref_isassigned && nargs == 3) { + else if (f == BUILTIN(memoryref_isassigned) && nargs == 3) { const jl_cgval_t &ref = argv[1]; jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { @@ -4384,7 +4358,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } if (!isboxed) elem = emit_ptrgep(ctx, elem, layout->first_ptr * sizeof(void*)); - // emit this using the same type as jl_builtin_memoryrefget + // emit this using the same type as BUILTIN(memoryrefget) // so that LLVM may be able to load-load forward them and fold the result auto tbaa = isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf; jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); @@ -4413,7 +4387,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } - else if (f == jl_builtin_getfield && (nargs == 2 || nargs == 3 || nargs == 4)) { + else if (f == BUILTIN(getfield) && (nargs == 2 || nargs == 3 || nargs == 4)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; enum jl_memory_order order = jl_memory_order_unspecified; @@ -4573,7 +4547,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - else if (f == jl_builtin_getglobal && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(getglobal) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; enum jl_memory_order order = jl_memory_order_unspecified; @@ -4605,23 +4579,23 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } - else if ((f == jl_builtin_setglobal && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_swapglobal && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_replaceglobal && (nargs == 4 || nargs == 5 || nargs == 6)) || - (f == jl_builtin_modifyglobal && (nargs == 4 || nargs == 5)) || - (f == jl_builtin_setglobalonce && (nargs == 3 || nargs == 4 || nargs == 5))) { + else if ((f == BUILTIN(setglobal) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(swapglobal) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(replaceglobal) && (nargs == 4 || nargs == 5 || nargs == 6)) || + (f == BUILTIN(modifyglobal) && (nargs == 4 || nargs == 5)) || + (f == BUILTIN(setglobalonce) && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opglobal(ctx, ret, f, argv, nargs, nullptr); } - else if ((f == jl_builtin_setfield && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_swapfield && (nargs == 3 || nargs == 4)) || - (f == jl_builtin_replacefield && (nargs == 4 || nargs == 5 || nargs == 6)) || - (f == jl_builtin_modifyfield && (nargs == 4 || nargs == 5)) || - (f == jl_builtin_setfieldonce && (nargs == 3 || nargs == 4 || nargs == 5))) { + else if ((f == BUILTIN(setfield) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(swapfield) && (nargs == 3 || nargs == 4)) || + (f == BUILTIN(replacefield) && (nargs == 4 || nargs == 5 || nargs == 6)) || + (f == BUILTIN(modifyfield) && (nargs == 4 || nargs == 5)) || + (f == BUILTIN(setfieldonce) && (nargs == 3 || nargs == 4 || nargs == 5))) { return emit_f_opfield(ctx, ret, f, argv, nargs, nullptr); } - else if (f == jl_builtin_nfields && nargs == 1) { + else if (f == BUILTIN(nfields) && nargs == 1) { const jl_cgval_t &obj = argv[1]; if (ctx.vaSlot > 0) { // optimize VA tuple @@ -4653,7 +4627,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_fieldtype && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(fieldtype) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &typ = argv[1]; const jl_cgval_t &fld = argv[2]; if ((jl_is_type_type(typ.typ) && jl_is_concrete_type(jl_tparam0(typ.typ))) || @@ -4678,7 +4652,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_sizeof && nargs == 1) { + else if (f == BUILTIN(sizeof) && nargs == 1) { const jl_cgval_t &obj = argv[1]; jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(obj.typ); assert(jl_string_type->name->mutabl); @@ -4723,7 +4697,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_apply_type && nargs > 0) { + else if (f == BUILTIN(apply_type) && nargs > 0) { if (jl_is_method(ctx.linfo->def.method)) { // don't bother codegen constant-folding for toplevel. jl_value_t *ty = static_apply_type(ctx, argv, nargs + 1); @@ -4737,7 +4711,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } - else if (f == jl_builtin_isdefinedglobal && (nargs == 2 || nargs == 3 || nargs == 4)) { + else if (f == BUILTIN(isdefinedglobal) && (nargs == 2 || nargs == 3 || nargs == 4)) { const jl_cgval_t &mod = argv[1]; const jl_cgval_t &sym = argv[2]; bool allow_import = true; @@ -4773,7 +4747,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_isdefined && (nargs == 2 || nargs == 3)) { + else if (f == BUILTIN(isdefined) && (nargs == 2 || nargs == 3)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; jl_datatype_t *stt = (jl_datatype_t*)obj.typ; @@ -4889,7 +4863,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_current_scope && (nargs == 0)) { + else if (f == BUILTIN(current_scope) && (nargs == 0)) { jl_aliasinfo_t scope_ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); Instruction *v = scope_ai.decorateInst( ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, get_scope_field(ctx), ctx.types().alignof_ptr)); @@ -4897,18 +4871,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_donotdelete) { + else if (f == BUILTIN(donotdelete)) { // For now we emit this as a vararg call to the builtin // (which doesn't look at the arguments). In the future, // this should be an LLVM builtin. - auto it = builtin_func_map().find(jl_f_donotdelete_addr); - if (it == builtin_func_map().end()) { - return false; - } - *ret = mark_julia_const(ctx, jl_nothing); FunctionType *Fty = FunctionType::get(getVoidTy(ctx.builder.getContext()), true); - Function *dnd = prepare_call(it->second); + Function *dnd = prepare_call(jldnd_func); SmallVector call_args; for (size_t i = 1; i <= nargs; ++i) { @@ -4927,7 +4896,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_compilerbarrier && (nargs == 2)) { + else if (f == BUILTIN(compilerbarrier) && (nargs == 2)) { emit_typecheck(ctx, argv[1], (jl_value_t*)jl_symbol_type, "compilerbarrier"); *ret = argv[2]; return true; @@ -5350,22 +5319,22 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ if (f.constant) { jl_cgval_t ret; auto it = builtin_func_map().end(); - if (f.constant == jl_builtin_modifyfield) { - if (emit_f_opfield(ctx, &ret, jl_builtin_modifyfield, argv, nargs - 1, &lival)) + if (f.constant == BUILTIN(modifyfield)) { + if (emit_f_opfield(ctx, &ret, BUILTIN(modifyfield), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_modifyfield_addr); + it = builtin_func_map().find(f.constant); assert(it != builtin_func_map().end()); } - else if (f.constant == jl_builtin_modifyglobal) { - if (emit_f_opglobal(ctx, &ret, jl_builtin_modifyglobal, argv, nargs - 1, &lival)) + else if (f.constant == BUILTIN(modifyglobal)) { + if (emit_f_opglobal(ctx, &ret, BUILTIN(modifyglobal), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_modifyglobal_addr); + it = builtin_func_map().find(f.constant); assert(it != builtin_func_map().end()); } - else if (f.constant == jl_builtin_memoryrefmodify) { - if (emit_f_opmemory(ctx, &ret, jl_builtin_memoryrefmodify, argv, nargs - 1, &lival)) + else if (f.constant == BUILTIN(memoryrefmodify)) { + if (emit_f_opmemory(ctx, &ret, BUILTIN(memoryrefmodify), argv, nargs - 1, &lival)) return ret; - it = builtin_func_map().find(jl_f_memoryrefmodify_addr); + it = builtin_func_map().find(f.constant); assert(it != builtin_func_map().end()); } else if (jl_typetagis(f.constant, jl_intrinsic_type)) { @@ -5425,15 +5394,15 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo return jl_cgval_t(); } + // a couple intrinsics (really just llvmcall, though partly cglobal too) + // have non-standard (aka invalid) evaluation semantics, so we must handle these first if (f.constant && jl_typetagis(f.constant, jl_intrinsic_type)) { JL_I::intrinsic fi = (intrinsic)*(uint32_t*)jl_data_ptr(f.constant); return emit_intrinsic(ctx, fi, args, nargs - 1); } size_t n_generic_args = nargs; - SmallVector argv(n_generic_args); - argv[0] = f; for (size_t i = 1; i < nargs; ++i) { argv[i] = emit_expr(ctx, args[i]); @@ -5441,37 +5410,23 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo return jl_cgval_t(); // anything past here is unreachable } - if (jl_subtype(f.typ, (jl_value_t*)jl_builtin_type)) { - if (f.constant) { - if (f.constant == jl_builtin_ifelse && nargs == 4) - return emit_ifelse(ctx, argv[1], argv[2], argv[3], rt); - jl_cgval_t result; - bool handled = emit_builtin_call(ctx, &result, f.constant, argv, nargs - 1, rt, ex, is_promotable); - if (handled) - return result; - jl_fptr_args_t builtin_fptr = jl_get_builtin_fptr((jl_datatype_t*)jl_typeof(f.constant)); - // special case for some known builtin not handled by emit_builtin_call - auto it = builtin_func_map().find(builtin_fptr); - if (it != builtin_func_map().end()) { - Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); - setName(ctx.emission_context, ret, it->second->name + "_ret"); - return mark_julia_type(ctx, ret, true, rt); - } - } - Value *fptr; - JuliaFunction<> *cc; - if (f.typ == (jl_value_t*)jl_intrinsic_type) { - fptr = prepare_call(jlintrinsic_func); - cc = julia_call3; - } - else { - fptr = ctx.builder.CreateCall(prepare_call(jlgetbuiltinfptr_func), {emit_typeof(ctx, f)}); - cc = julia_call; - } - Value *ret = emit_jlcall(ctx, fptr, nullptr, argv, nargs, cc); + if (f.typ == (jl_value_t*)jl_intrinsic_type) { + Value *ret = emit_jlcall(ctx, prepare_call(jlintrinsic_func), nullptr, argv, nargs, julia_call3); setName(ctx.emission_context, ret, "Builtin_ret"); return mark_julia_type(ctx, ret, true, rt); } + else if (f.constant && jl_isa(f.constant, (jl_value_t*)jl_builtin_type)) { + jl_cgval_t result; + bool handled = emit_builtin_call(ctx, &result, f.constant, argv, nargs - 1, rt, ex, is_promotable); + if (handled) + return result; + auto it = builtin_func_map().find(f.constant); + if (it != builtin_func_map().end()) { + Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); + setName(ctx.emission_context, ret, it->second->name + "_ret"); + return mark_julia_type(ctx, ret, true, rt); + } + } // handle calling an OpaqueClosure if (jl_is_concrete_type(f.typ) && jl_subtype(f.typ, (jl_value_t*)jl_opaque_closure_type)) { @@ -9845,7 +9800,6 @@ static void init_jit_functions(void) { add_named_global("jl_fptr_args", jl_fptr_args_addr); add_named_global("jl_fptr_sparam", jl_fptr_sparam_addr); - add_named_global("jl_f_opaque_closure_call", &jl_f_opaque_closure_call); add_named_global(jl_small_typeof_var, &jl_small_typeof); add_named_global(jlstack_chk_guard_var, &__stack_chk_guard); add_named_global(jlRTLD_DEFAULT_var, &jl_RTLD_DEFAULT_handle); @@ -9882,10 +9836,8 @@ static void init_jit_functions(void) add_named_global(jlcheckassign_func, &jl_checked_assignment); add_named_global(jlcheckbpwritable_func, &jl_check_binding_currently_writable); add_named_global(jlboundp_func, &jl_boundp); - for (auto it : builtin_func_map()) - add_named_global(it.second, it.first); - add_named_global(jlintrinsic_func, &jl_f_intrinsic_call); - add_named_global(jlgetbuiltinfptr_func, &jl_get_builtin_fptr); + for (int i = 0; i < jl_n_builtins; i++) + add_named_global(jl_builtin_f_names[i], jl_builtin_f_addrs[i]); add_named_global(jlapplygeneric_func, &jl_apply_generic); add_named_global(jlinvoke_func, &jl_invoke); add_named_global(jltopeval_func, &jl_toplevel_eval); diff --git a/src/common_symbols2.inc b/src/common_symbols2.inc index 2a6990bac52ff..e9c070ee8da6a 100644 --- a/src/common_symbols2.inc +++ b/src/common_symbols2.inc @@ -97,7 +97,7 @@ jl_symbol("pointerref"), jl_symbol("multidimensional.jl"), jl_symbol("Generator"), jl_symbol("leave"), -jl_symbol("memoryref"), +jl_symbol("memoryrefnew"), jl_symbol("show.jl"), jl_symbol("pointer_from_objref"), jl_symbol("memoryrefget"), diff --git a/src/gf.c b/src/gf.c index 2f4b838f04908..b130fbe05ca6d 100644 --- a/src/gf.c +++ b/src/gf.c @@ -288,16 +288,8 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *typ // ----- MethodInstance specialization instantiation ----- // -jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED +void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *sname, jl_fptr_args_t fptr) JL_GC_DISABLED { - jl_sym_t *sname = jl_symbol(name); - if (dt == NULL) { - // Builtins are specially considered available from world 0 - jl_value_t *f = jl_new_generic_function_with_supertype(sname, jl_core_module, jl_builtin_type, 0); - jl_set_initial_const(jl_core_module, sname, f, 0); - dt = (jl_datatype_t*)jl_typeof(f); - } - jl_method_t *m = jl_new_method_uninit(jl_core_module); m->name = sname; m->module = jl_core_module; @@ -335,7 +327,6 @@ jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_a mt->frozen = 1; JL_GC_POP(); - return dt; } // only relevant for bootstrapping. otherwise fairly broken. @@ -3110,6 +3101,7 @@ JL_DLLEXPORT const jl_callptr_t jl_fptr_const_return_addr = &jl_fptr_const_retur JL_DLLEXPORT const jl_callptr_t jl_fptr_sparam_addr = &jl_fptr_sparam; +JL_CALLABLE(jl_f_opaque_closure_call); JL_DLLEXPORT const jl_callptr_t jl_f_opaque_closure_call_addr = (jl_callptr_t)&jl_f_opaque_closure_call; JL_DLLEXPORT const jl_callptr_t jl_fptr_wait_for_compiled_addr = &jl_fptr_wait_for_compiled; diff --git a/src/init.c b/src/init.c index 6d5212cf8d370..cf432a6bd6047 100644 --- a/src/init.c +++ b/src/init.c @@ -23,9 +23,7 @@ #include "julia.h" #include "julia_internal.h" -#define DEFINE_BUILTIN_GLOBALS #include "builtin_proto.h" -#undef DEFINE_BUILTIN_GLOBALS #include "threading.h" #include "julia_assert.h" #include "processor.h" diff --git a/src/julia_internal.h b/src/julia_internal.h index 864097d8d22fe..5d418ceb2051c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -21,6 +21,9 @@ #include #include +#define STR(x) #x +#define XSTR(x) STR(x) + #if !defined(_WIN32) #include #else @@ -770,16 +773,11 @@ JL_DLLEXPORT void jl_typeassert(jl_value_t *x, jl_value_t *t); #define JL_CALLABLE(name) \ JL_DLLEXPORT jl_value_t *name(jl_value_t *F, jl_value_t **args, uint32_t nargs) -JL_CALLABLE(jl_f_svec); JL_CALLABLE(jl_f_tuple); -JL_CALLABLE(jl_f_intrinsic_call); -JL_CALLABLE(jl_f_opaque_closure_call); void jl_install_default_signal_handlers(void); void restore_signals(void); void jl_install_thread_signal_handler(jl_ptls_t ptls); -JL_DLLEXPORT jl_fptr_args_t jl_get_builtin_fptr(jl_datatype_t *dt); - extern uv_loop_t *jl_io_loop; JL_DLLEXPORT void jl_uv_flush(uv_stream_t *stream); @@ -806,7 +804,7 @@ jl_tupletype_t *jl_lookup_arg_tuple_type(jl_value_t *arg1 JL_PROPAGATES_ROOT, jl JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry); jl_typemap_entry_t *jl_method_table_add(jl_methtable_t *mt, jl_method_t *method, jl_tupletype_t *simpletype); -jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED; +void jl_mk_builtin_func(jl_datatype_t *dt, jl_sym_t *name, jl_fptr_args_t fptr) JL_GC_DISABLED; int jl_obviously_unequal(jl_value_t *a, jl_value_t *b); int jl_has_bound_typevars(jl_value_t *v, jl_typeenv_t *env) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_array_t *jl_find_free_typevars(jl_value_t *v); diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index cfdb8eb5b1a99..1dece818fa998 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -13,9 +13,6 @@ #include "julia.h" -#define STR(csym) #csym -#define XSTR(csym) STR(csym) - static constexpr std::nullopt_t None = std::nullopt; enum AddressSpace { diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 30a6e0b37fe50..dbafc12aeff94 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -18,6 +18,9 @@ #include "julia_assert.h" #include "llvm-pass-helpers.h" +#define STR(csym) #csym +#define XSTR(csym) STR(csym) + using namespace llvm; JuliaPassContext::JuliaPassContext() diff --git a/src/method.c b/src/method.c index b3ed63e810f77..aa4f438ede080 100644 --- a/src/method.c +++ b/src/method.c @@ -10,13 +10,12 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "builtin_proto.h" #ifdef __cplusplus extern "C" { #endif -extern jl_value_t *jl_builtin_getfield; -extern jl_value_t *jl_builtin_tuple; jl_methtable_t *jl_kwcall_mt; jl_method_t *jl_opaque_closure_method; @@ -225,7 +224,7 @@ static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *mod jl_sym_t *fe_sym = jl_globalref_name(fe); // look at some known called functions jl_binding_t *b = jl_get_binding(fe_mod, fe_sym); - if (jl_get_binding_value_if_const(b) == jl_builtin_tuple) { + if (jl_get_binding_value_if_const(b) == BUILTIN(tuple)) { size_t j; for (j = 1; j < nargs; j++) { if (!jl_is_quotenode(jl_exprarg(e, j))) diff --git a/src/signals-mach.c b/src/signals-mach.c index 05f1111bab6ae..b7057416ad407 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -146,8 +146,6 @@ static void jl_mach_gc_wait(jl_ptls_t ptls2, mach_port_t thread, int16_t tid) static mach_port_t segv_port = 0; -#define STR(x) #x -#define XSTR(x) STR(x) #define HANDLE_MACH_ERROR(msg, retval) \ if (retval != KERN_SUCCESS) { mach_error(msg XSTR(: __FILE__:__LINE__:), (retval)); abort(); } diff --git a/src/staticdata.c b/src/staticdata.c index 64f66e87252aa..a8ebf1d9bc345 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -116,228 +116,177 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 197 +#define NUM_TAGS 152 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. -jl_value_t **const*const get_tags(void) { +static void get_tags(jl_value_t **tags[NUM_TAGS]) +{ // Make sure to keep an extra slot at the end to sentinel length - static void * _tags[NUM_TAGS] = {NULL}; - - // Lazyily-initialize this list - if (_tags[0] == NULL) { - unsigned int i = 0; -#define INSERT_TAG(sym) _tags[i++] = &(sym) - // builtin types - INSERT_TAG(jl_any_type); - INSERT_TAG(jl_symbol_type); - INSERT_TAG(jl_ssavalue_type); - INSERT_TAG(jl_datatype_type); - INSERT_TAG(jl_slotnumber_type); - INSERT_TAG(jl_simplevector_type); - INSERT_TAG(jl_array_type); - INSERT_TAG(jl_expr_type); - INSERT_TAG(jl_binding_type); - INSERT_TAG(jl_binding_partition_type); - INSERT_TAG(jl_globalref_type); - INSERT_TAG(jl_string_type); - INSERT_TAG(jl_module_type); - INSERT_TAG(jl_tvar_type); - INSERT_TAG(jl_method_instance_type); - INSERT_TAG(jl_method_type); - INSERT_TAG(jl_code_instance_type); - INSERT_TAG(jl_linenumbernode_type); - INSERT_TAG(jl_lineinfonode_type); - INSERT_TAG(jl_gotonode_type); - INSERT_TAG(jl_quotenode_type); - INSERT_TAG(jl_gotoifnot_type); - INSERT_TAG(jl_enternode_type); - INSERT_TAG(jl_argument_type); - INSERT_TAG(jl_returnnode_type); - INSERT_TAG(jl_const_type); - INSERT_TAG(jl_partial_struct_type); - INSERT_TAG(jl_partial_opaque_type); - INSERT_TAG(jl_interconditional_type); - INSERT_TAG(jl_method_match_type); - INSERT_TAG(jl_pinode_type); - INSERT_TAG(jl_phinode_type); - INSERT_TAG(jl_phicnode_type); - INSERT_TAG(jl_upsilonnode_type); - INSERT_TAG(jl_type_type); - INSERT_TAG(jl_bottom_type); - INSERT_TAG(jl_ref_type); - INSERT_TAG(jl_pointer_type); - INSERT_TAG(jl_llvmpointer_type); - INSERT_TAG(jl_vararg_type); - INSERT_TAG(jl_abstractarray_type); - INSERT_TAG(jl_densearray_type); - INSERT_TAG(jl_nothing_type); - INSERT_TAG(jl_function_type); - INSERT_TAG(jl_typeofbottom_type); - INSERT_TAG(jl_unionall_type); - INSERT_TAG(jl_typename_type); - INSERT_TAG(jl_builtin_type); - INSERT_TAG(jl_code_info_type); - INSERT_TAG(jl_opaque_closure_type); - INSERT_TAG(jl_task_type); - INSERT_TAG(jl_uniontype_type); - INSERT_TAG(jl_abstractstring_type); - INSERT_TAG(jl_array_any_type); - INSERT_TAG(jl_intrinsic_type); - INSERT_TAG(jl_methtable_type); - INSERT_TAG(jl_typemap_level_type); - INSERT_TAG(jl_typemap_entry_type); - INSERT_TAG(jl_voidpointer_type); - INSERT_TAG(jl_uint8pointer_type); - INSERT_TAG(jl_newvarnode_type); - INSERT_TAG(jl_anytuple_type_type); - INSERT_TAG(jl_anytuple_type); - INSERT_TAG(jl_namedtuple_type); - INSERT_TAG(jl_emptytuple_type); - INSERT_TAG(jl_array_symbol_type); - INSERT_TAG(jl_array_uint8_type); - INSERT_TAG(jl_array_uint32_type); - INSERT_TAG(jl_array_int32_type); - INSERT_TAG(jl_array_uint64_type); - INSERT_TAG(jl_int32_type); - INSERT_TAG(jl_int64_type); - INSERT_TAG(jl_bool_type); - INSERT_TAG(jl_uint8_type); - INSERT_TAG(jl_uint16_type); - INSERT_TAG(jl_uint32_type); - INSERT_TAG(jl_uint64_type); - INSERT_TAG(jl_char_type); - INSERT_TAG(jl_weakref_type); - INSERT_TAG(jl_int8_type); - INSERT_TAG(jl_int16_type); - INSERT_TAG(jl_float16_type); - INSERT_TAG(jl_float32_type); - INSERT_TAG(jl_float64_type); - INSERT_TAG(jl_bfloat16_type); - INSERT_TAG(jl_floatingpoint_type); - INSERT_TAG(jl_number_type); - INSERT_TAG(jl_signed_type); - INSERT_TAG(jl_pair_type); - INSERT_TAG(jl_genericmemory_type); - INSERT_TAG(jl_memory_any_type); - INSERT_TAG(jl_memory_uint8_type); - INSERT_TAG(jl_memory_uint16_type); - INSERT_TAG(jl_memory_uint32_type); - INSERT_TAG(jl_memory_uint64_type); - INSERT_TAG(jl_genericmemoryref_type); - INSERT_TAG(jl_memoryref_any_type); - INSERT_TAG(jl_memoryref_uint8_type); - INSERT_TAG(jl_addrspace_type); - INSERT_TAG(jl_addrspace_typename); - INSERT_TAG(jl_addrspacecore_type); - INSERT_TAG(jl_debuginfo_type); - INSERT_TAG(jl_abioverride_type); - - // special typenames - INSERT_TAG(jl_tuple_typename); - INSERT_TAG(jl_pointer_typename); - INSERT_TAG(jl_llvmpointer_typename); - INSERT_TAG(jl_array_typename); - INSERT_TAG(jl_type_typename); - INSERT_TAG(jl_namedtuple_typename); - INSERT_TAG(jl_vecelement_typename); - INSERT_TAG(jl_opaque_closure_typename); - INSERT_TAG(jl_genericmemory_typename); - INSERT_TAG(jl_genericmemoryref_typename); - - // special exceptions - INSERT_TAG(jl_errorexception_type); - INSERT_TAG(jl_argumenterror_type); - INSERT_TAG(jl_typeerror_type); - INSERT_TAG(jl_methoderror_type); - INSERT_TAG(jl_loaderror_type); - INSERT_TAG(jl_initerror_type); - INSERT_TAG(jl_undefvarerror_type); - INSERT_TAG(jl_fielderror_type); - INSERT_TAG(jl_stackovf_exception); - INSERT_TAG(jl_diverror_exception); - INSERT_TAG(jl_interrupt_exception); - INSERT_TAG(jl_boundserror_type); - INSERT_TAG(jl_memory_exception); - INSERT_TAG(jl_undefref_exception); - INSERT_TAG(jl_readonlymemory_exception); - INSERT_TAG(jl_atomicerror_type); - INSERT_TAG(jl_missingcodeerror_type); - INSERT_TAG(jl_precompilable_error); - INSERT_TAG(jl_trimfailure_type); - - // other special values - INSERT_TAG(jl_emptysvec); - INSERT_TAG(jl_emptytuple); - INSERT_TAG(jl_false); - INSERT_TAG(jl_true); - INSERT_TAG(jl_an_empty_string); - INSERT_TAG(jl_an_empty_vec_any); - INSERT_TAG(jl_an_empty_memory_any); - INSERT_TAG(jl_module_init_order); - INSERT_TAG(jl_core_module); - INSERT_TAG(jl_base_module); - INSERT_TAG(jl_main_module); - INSERT_TAG(jl_top_module); - INSERT_TAG(jl_typeinf_func); - INSERT_TAG(jl_type_type_mt); - INSERT_TAG(jl_nonfunction_mt); - INSERT_TAG(jl_kwcall_mt); - INSERT_TAG(jl_kwcall_func); - INSERT_TAG(jl_opaque_closure_method); - INSERT_TAG(jl_nulldebuginfo); - - // some Core.Builtin Functions that we want to be able to reference: - INSERT_TAG(jl_builtin_throw); - INSERT_TAG(jl_builtin_is); - INSERT_TAG(jl_builtin_typeof); - INSERT_TAG(jl_builtin_sizeof); - INSERT_TAG(jl_builtin_issubtype); - INSERT_TAG(jl_builtin_isa); - INSERT_TAG(jl_builtin_typeassert); - INSERT_TAG(jl_builtin__apply_iterate); - INSERT_TAG(jl_builtin_isdefined); - INSERT_TAG(jl_builtin_nfields); - INSERT_TAG(jl_builtin_tuple); - INSERT_TAG(jl_builtin_svec); - INSERT_TAG(jl_builtin_getfield); - INSERT_TAG(jl_builtin_setfield); - INSERT_TAG(jl_builtin_swapfield); - INSERT_TAG(jl_builtin_modifyfield); - INSERT_TAG(jl_builtin_replacefield); - INSERT_TAG(jl_builtin_setfieldonce); - INSERT_TAG(jl_builtin_fieldtype); - INSERT_TAG(jl_builtin_memorynew); - INSERT_TAG(jl_builtin_memoryref); - INSERT_TAG(jl_builtin_memoryrefoffset); - INSERT_TAG(jl_builtin_memoryrefget); - INSERT_TAG(jl_builtin_memoryrefset); - INSERT_TAG(jl_builtin_memoryref_isassigned); - INSERT_TAG(jl_builtin_memoryrefswap); - INSERT_TAG(jl_builtin_memoryrefmodify); - INSERT_TAG(jl_builtin_memoryrefreplace); - INSERT_TAG(jl_builtin_memoryrefsetonce); - INSERT_TAG(jl_builtin_apply_type); - INSERT_TAG(jl_builtin_applicable); - INSERT_TAG(jl_builtin_invoke); - INSERT_TAG(jl_builtin__expr); - INSERT_TAG(jl_builtin_ifelse); - INSERT_TAG(jl_builtin__typebody); - INSERT_TAG(jl_builtin_donotdelete); - INSERT_TAG(jl_builtin_compilerbarrier); - INSERT_TAG(jl_builtin_getglobal); - INSERT_TAG(jl_builtin_setglobal); - INSERT_TAG(jl_builtin_isdefinedglobal); - INSERT_TAG(jl_builtin_swapglobal); - INSERT_TAG(jl_builtin_modifyglobal); - INSERT_TAG(jl_builtin_replaceglobal); - INSERT_TAG(jl_builtin_setglobalonce); - INSERT_TAG(jl_builtin_current_scope); - // n.b. must update NUM_TAGS when you add something here + unsigned int i = 0; +#define INSERT_TAG(sym) tags[i++] = (jl_value_t**)&(sym) + // builtin types + INSERT_TAG(jl_any_type); + INSERT_TAG(jl_symbol_type); + INSERT_TAG(jl_ssavalue_type); + INSERT_TAG(jl_datatype_type); + INSERT_TAG(jl_slotnumber_type); + INSERT_TAG(jl_simplevector_type); + INSERT_TAG(jl_array_type); + INSERT_TAG(jl_expr_type); + INSERT_TAG(jl_binding_type); + INSERT_TAG(jl_binding_partition_type); + INSERT_TAG(jl_globalref_type); + INSERT_TAG(jl_string_type); + INSERT_TAG(jl_module_type); + INSERT_TAG(jl_tvar_type); + INSERT_TAG(jl_method_instance_type); + INSERT_TAG(jl_method_type); + INSERT_TAG(jl_code_instance_type); + INSERT_TAG(jl_linenumbernode_type); + INSERT_TAG(jl_lineinfonode_type); + INSERT_TAG(jl_gotonode_type); + INSERT_TAG(jl_quotenode_type); + INSERT_TAG(jl_gotoifnot_type); + INSERT_TAG(jl_enternode_type); + INSERT_TAG(jl_argument_type); + INSERT_TAG(jl_returnnode_type); + INSERT_TAG(jl_const_type); + INSERT_TAG(jl_partial_struct_type); + INSERT_TAG(jl_partial_opaque_type); + INSERT_TAG(jl_interconditional_type); + INSERT_TAG(jl_method_match_type); + INSERT_TAG(jl_pinode_type); + INSERT_TAG(jl_phinode_type); + INSERT_TAG(jl_phicnode_type); + INSERT_TAG(jl_upsilonnode_type); + INSERT_TAG(jl_type_type); + INSERT_TAG(jl_bottom_type); + INSERT_TAG(jl_ref_type); + INSERT_TAG(jl_pointer_type); + INSERT_TAG(jl_llvmpointer_type); + INSERT_TAG(jl_vararg_type); + INSERT_TAG(jl_abstractarray_type); + INSERT_TAG(jl_densearray_type); + INSERT_TAG(jl_nothing_type); + INSERT_TAG(jl_function_type); + INSERT_TAG(jl_typeofbottom_type); + INSERT_TAG(jl_unionall_type); + INSERT_TAG(jl_typename_type); + INSERT_TAG(jl_builtin_type); + INSERT_TAG(jl_code_info_type); + INSERT_TAG(jl_opaque_closure_type); + INSERT_TAG(jl_task_type); + INSERT_TAG(jl_uniontype_type); + INSERT_TAG(jl_abstractstring_type); + INSERT_TAG(jl_array_any_type); + INSERT_TAG(jl_intrinsic_type); + INSERT_TAG(jl_methtable_type); + INSERT_TAG(jl_typemap_level_type); + INSERT_TAG(jl_typemap_entry_type); + INSERT_TAG(jl_voidpointer_type); + INSERT_TAG(jl_uint8pointer_type); + INSERT_TAG(jl_newvarnode_type); + INSERT_TAG(jl_anytuple_type_type); + INSERT_TAG(jl_anytuple_type); + INSERT_TAG(jl_namedtuple_type); + INSERT_TAG(jl_emptytuple_type); + INSERT_TAG(jl_array_symbol_type); + INSERT_TAG(jl_array_uint8_type); + INSERT_TAG(jl_array_uint32_type); + INSERT_TAG(jl_array_int32_type); + INSERT_TAG(jl_array_uint64_type); + INSERT_TAG(jl_int32_type); + INSERT_TAG(jl_int64_type); + INSERT_TAG(jl_bool_type); + INSERT_TAG(jl_uint8_type); + INSERT_TAG(jl_uint16_type); + INSERT_TAG(jl_uint32_type); + INSERT_TAG(jl_uint64_type); + INSERT_TAG(jl_char_type); + INSERT_TAG(jl_weakref_type); + INSERT_TAG(jl_int8_type); + INSERT_TAG(jl_int16_type); + INSERT_TAG(jl_float16_type); + INSERT_TAG(jl_float32_type); + INSERT_TAG(jl_float64_type); + INSERT_TAG(jl_bfloat16_type); + INSERT_TAG(jl_floatingpoint_type); + INSERT_TAG(jl_number_type); + INSERT_TAG(jl_signed_type); + INSERT_TAG(jl_pair_type); + INSERT_TAG(jl_genericmemory_type); + INSERT_TAG(jl_memory_any_type); + INSERT_TAG(jl_memory_uint8_type); + INSERT_TAG(jl_memory_uint16_type); + INSERT_TAG(jl_memory_uint32_type); + INSERT_TAG(jl_memory_uint64_type); + INSERT_TAG(jl_genericmemoryref_type); + INSERT_TAG(jl_memoryref_any_type); + INSERT_TAG(jl_memoryref_uint8_type); + INSERT_TAG(jl_addrspace_type); + INSERT_TAG(jl_addrspace_typename); + INSERT_TAG(jl_addrspacecore_type); + INSERT_TAG(jl_debuginfo_type); + INSERT_TAG(jl_abioverride_type); + + // special typenames + INSERT_TAG(jl_tuple_typename); + INSERT_TAG(jl_pointer_typename); + INSERT_TAG(jl_llvmpointer_typename); + INSERT_TAG(jl_array_typename); + INSERT_TAG(jl_type_typename); + INSERT_TAG(jl_namedtuple_typename); + INSERT_TAG(jl_vecelement_typename); + INSERT_TAG(jl_opaque_closure_typename); + INSERT_TAG(jl_genericmemory_typename); + INSERT_TAG(jl_genericmemoryref_typename); + + // special exceptions + INSERT_TAG(jl_errorexception_type); + INSERT_TAG(jl_argumenterror_type); + INSERT_TAG(jl_typeerror_type); + INSERT_TAG(jl_methoderror_type); + INSERT_TAG(jl_loaderror_type); + INSERT_TAG(jl_initerror_type); + INSERT_TAG(jl_undefvarerror_type); + INSERT_TAG(jl_fielderror_type); + INSERT_TAG(jl_stackovf_exception); + INSERT_TAG(jl_diverror_exception); + INSERT_TAG(jl_interrupt_exception); + INSERT_TAG(jl_boundserror_type); + INSERT_TAG(jl_memory_exception); + INSERT_TAG(jl_undefref_exception); + INSERT_TAG(jl_readonlymemory_exception); + INSERT_TAG(jl_atomicerror_type); + INSERT_TAG(jl_missingcodeerror_type); + INSERT_TAG(jl_precompilable_error); + INSERT_TAG(jl_trimfailure_type); + + // other special values + INSERT_TAG(jl_emptysvec); + INSERT_TAG(jl_emptytuple); + INSERT_TAG(jl_false); + INSERT_TAG(jl_true); + INSERT_TAG(jl_an_empty_string); + INSERT_TAG(jl_an_empty_vec_any); + INSERT_TAG(jl_an_empty_memory_any); + INSERT_TAG(jl_module_init_order); + INSERT_TAG(jl_core_module); + INSERT_TAG(jl_base_module); + INSERT_TAG(jl_main_module); + INSERT_TAG(jl_top_module); + INSERT_TAG(jl_typeinf_func); + INSERT_TAG(jl_type_type_mt); + INSERT_TAG(jl_nonfunction_mt); + INSERT_TAG(jl_kwcall_mt); + INSERT_TAG(jl_kwcall_func); + INSERT_TAG(jl_opaque_closure_method); + INSERT_TAG(jl_nulldebuginfo); + // n.b. must update NUM_TAGS when you add something here #undef INSERT_TAG - assert(i == NUM_TAGS - 1); - } - return (jl_value_t**const*const) _tags; + assert(i == NUM_TAGS - 1); + tags[i] = NULL; } // hash of definitions for predefined tagged object @@ -373,12 +322,12 @@ static uintptr_t img_max; // HT_NOTFOUND is a valid integer ID, so we store the integer ids mangled. // This pair of functions mangles/demanges -static size_t from_seroder_entry(void *entry) +static size_t from_seroder_entry(void *entry) JL_NOTSAFEPOINT { return (size_t)((char*)entry - (char*)HT_NOTFOUND - 1); } -static void *to_seroder_entry(size_t idx) +static void *to_seroder_entry(size_t idx) JL_NOTSAFEPOINT { return (void*)((char*)HT_NOTFOUND + 1 + idx); } @@ -386,7 +335,7 @@ static void *to_seroder_entry(size_t idx) static htable_t new_methtables; static size_t precompilation_world; -static int ptr_cmp(const void *l, const void *r) +static int ptr_cmp(const void *l, const void *r) JL_NOTSAFEPOINT { uintptr_t left = *(const uintptr_t*)l; uintptr_t right = *(const uintptr_t*)r; @@ -394,7 +343,7 @@ static int ptr_cmp(const void *l, const void *r) } // Build an eytzinger tree from a sorted array -static int eytzinger(uintptr_t *src, uintptr_t *dest, size_t i, size_t k, size_t n) +static int eytzinger(uintptr_t *src, uintptr_t *dest, size_t i, size_t k, size_t n) JL_NOTSAFEPOINT { if (k <= n) { i = eytzinger(src, dest, i, 2 * k, n); @@ -434,7 +383,7 @@ static size_t eyt_obj_idx(jl_value_t *obj) JL_NOTSAFEPOINT } //used in staticdata.c after we add an image -void rebuild_image_blob_tree(void) +void rebuild_image_blob_tree(void) JL_NOTSAFEPOINT { size_t inc = 1 + jl_linkage_blobs.len - eytzinger_image_tree.len; assert(eytzinger_idxs.len == eytzinger_image_tree.len); @@ -510,33 +459,15 @@ JL_DLLEXPORT jl_value_t *jl_object_top_module(jl_value_t* v) JL_NOTSAFEPOINT } // hash of definitions for predefined function pointers +// (reverse is jl_builtin_f_addrs) static htable_t fptr_to_id; + void *native_functions; // opaque jl_native_code_desc_t blob used for fetching data from LLVM // table of struct field addresses to rewrite during saving static htable_t field_replace; static htable_t bits_replace; -// array of definitions for the predefined function pointers -// (reverse of fptr_to_id) -// This is a manually constructed dual of the fvars array, which would be produced by codegen for Julia code, for C. -static const jl_fptr_args_t id_to_fptrs[] = { - &jl_f_throw, &jl_f_throw_methoderror, &jl_f_is, &jl_f_typeof, &jl_f_issubtype, &jl_f_isa, - &jl_f_typeassert, &jl_f__apply_iterate, - &jl_f_invokelatest, &jl_f_invoke_in_world, &jl_f__call_in_world_total, &jl_f_isdefined, &jl_f_isdefinedglobal, - &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, - &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, &jl_f_setfieldonce, - &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, &jl_f_apply_type, &jl_f_memorynew, - &jl_f_memoryref, &jl_f_memoryrefoffset, &jl_f_memoryrefget, &jl_f_memoryref_isassigned, - &jl_f_memoryrefset, &jl_f_memoryrefswap, &jl_f_memoryrefmodify, &jl_f_memoryrefreplace, &jl_f_memoryrefsetonce, - &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, - &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, - &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f__defaultctors, - &jl_f_opaque_closure_call, &jl_f_donotdelete, &jl_f_compilerbarrier, &jl_f_get_binding_type, - &jl_f_getglobal, &jl_f_setglobal, &jl_f_swapglobal, &jl_f_modifyglobal, &jl_f_replaceglobal, &jl_f_setglobalonce, - &jl_f_finalizer, &jl_f__compute_sparams, &jl_f__svec_ref, - &jl_f_current_scope, - NULL }; typedef struct { ios_t *s; // the main stream @@ -1242,7 +1173,8 @@ static void write_pointer(ios_t *s) JL_NOTSAFEPOINT } // Records the buildid holding `v` and returns the tagged offset within the corresponding image -static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) { +static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_GC_DISABLED +{ size_t i = external_blob_index(v); if (i < n_linkage_blobs()) { // We found the sysimg/pkg that this item links against @@ -1271,7 +1203,7 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ // but symbols, small integers, and a couple of special items (`nothing` and the root Task) // have special handling. #define backref_id(s, v, link_ids) _backref_id(s, (jl_value_t*)(v), link_ids) -static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_NOTSAFEPOINT +static uintptr_t _backref_id(jl_serializer_state *s, jl_value_t *v, jl_array_t *link_ids) JL_GC_DISABLED { assert(v != NULL && "cannot get backref to NULL object"); if (jl_is_symbol(v)) { @@ -1484,7 +1416,7 @@ static void record_memoryrefs_inside(jl_serializer_state *s, jl_datatype_t *t, s } } -static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFEPOINT +static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_GC_DISABLED { for (size_t i = 0; i < globals->len; i++) jl_queue_for_serialization(s, globals->items[i]); @@ -2126,7 +2058,7 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) case FunctionRef: if (offset & BuiltinFunctionTag) { offset &= ~BuiltinFunctionTag; - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer id"); + assert(offset < jl_n_builtins && "unknown function pointer id"); } else { assert(offset < JL_API_MAX && "unknown function pointer id"); @@ -2181,8 +2113,8 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas case FunctionRef: if (offset & BuiltinFunctionTag) { offset &= ~BuiltinFunctionTag; - assert(offset < sizeof(id_to_fptrs) / sizeof(*id_to_fptrs) && "unknown function pointer ID"); - return (uintptr_t)id_to_fptrs[offset]; + assert(offset < jl_n_builtins && "unknown function pointer ID"); + return (uintptr_t)jl_builtin_f_addrs[offset]; } switch ((jl_callingconv_t)offset) { case JL_API_BOXED: @@ -2395,7 +2327,7 @@ void gc_sweep_sysimg(void) // the image proper. For example, new methods added to external callables require // insertion into the appropriate method table. #define jl_write_value(s, v) _jl_write_value((s), (jl_value_t*)(v)) -static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) +static void _jl_write_value(jl_serializer_state *s, jl_value_t *v) JL_GC_DISABLED { if (v == NULL) { write_reloc_t(s->s, 0); @@ -2497,7 +2429,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) jl_register_fptrs(image->base, &fvars, linfos, img_fvars_max); } -static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_NOTSAFEPOINT +static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_GC_DISABLED { size_t len = globals->len + external_fns->len; ios_ensureroom(s->gvar_record, len * sizeof(reloc_t)); @@ -3160,10 +3092,10 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, nsym_tag = 0; htable_new(&symbol_table, 0); - htable_new(&fptr_to_id, sizeof(id_to_fptrs) / sizeof(*id_to_fptrs)); + htable_new(&fptr_to_id, jl_n_builtins); uintptr_t i; - for (i = 0; id_to_fptrs[i] != NULL; i++) { - ptrhash_put(&fptr_to_id, (void*)(uintptr_t)id_to_fptrs[i], (void*)(i + 2)); + for (i = 0; i < jl_n_builtins; i++) { + ptrhash_put(&fptr_to_id, (void*)(uintptr_t)jl_builtin_f_addrs[i], (void*)(i + 2)); } htable_new(&serialization_order, 25000); htable_new(&nullptrs, 0); @@ -3202,11 +3134,15 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_int32_type, 0); s.method_roots_list = NULL; htable_new(&s.method_roots_index, 0); + jl_value_t **_tags[NUM_TAGS]; + jl_value_t ***tags = s.incremental ? NULL : _tags; if (worklist) { s.method_roots_list = jl_alloc_vec_any(0); s.worklist_key = jl_worklist_key(worklist); } - jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; + else { + get_tags(_tags); + } if (worklist == NULL) { // empty!(Core.ARGS) @@ -3232,6 +3168,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *tag = *tags[i]; jl_queue_for_serialization(&s, tag); } + for (i = 0; i < jl_n_builtins; i++) + jl_queue_for_serialization(&s, jl_builtin_instances[i]); jl_queue_for_serialization(&s, s.ptls->root_task->tls); } else { @@ -3428,6 +3366,8 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *tag = *tags[i]; jl_write_value(&s, tag); } + for (i = 0; i < jl_n_builtins; i++) + jl_write_value(&s, jl_builtin_instances[i]); jl_write_value(&s, global_roots_list); jl_write_value(&s, global_roots_keyset); jl_write_value(&s, s.ptls->root_task->tls); @@ -3866,7 +3806,11 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, s.gvar_record = &gvar_record; s.fptr_record = &fptr_record; s.ptls = ct->ptls; - jl_value_t **const*const tags = get_tags(); + jl_value_t **_tags[NUM_TAGS]; + jl_value_t ***tags = s.incremental ? NULL : _tags; + if (!s.incremental) + get_tags(_tags); + htable_t new_dt_objs; htable_new(&new_dt_objs, 0); arraylist_new(&deser_sym, 0); @@ -3926,6 +3870,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl_value_t **tag = tags[i]; *tag = jl_read_value(&s); } + for (i = 0; i < jl_n_builtins; i++) + jl_builtin_instances[i] = jl_read_value(&s); #define XX(name) \ ijl_small_typeof[(jl_##name##_tag << 4) / sizeof(*ijl_small_typeof)] = jl_##name##_type; JL_SMALL_TYPEOF(XX) diff --git a/src/toplevel.c b/src/toplevel.c index 868c4e28a2849..64a559ac774e6 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -426,7 +426,7 @@ static void expr_attributes(jl_value_t *v, jl_array_t *body, int *has_ccall, int if (jl_is_intrinsic(called) && jl_unbox_int32(called) == (int)llvmcall) { *has_ccall = 1; } - if (called == jl_builtin__typebody) { // TODO: rely on latestworld instead of function callee detection here (or add it to jl_is_toplevel_only_expr) + if (called == BUILTIN(_typebody)) { // TODO: rely on latestworld instead of function callee detection here (or add it to jl_is_toplevel_only_expr) *has_defs = 1; } } @@ -705,7 +705,7 @@ static void jl_eval_throw(jl_module_t *m, jl_value_t *exc, const char *filename, { jl_value_t *throw_ex = (jl_value_t*)jl_exprn(jl_call_sym, 2); JL_GC_PUSH1(&throw_ex); - jl_exprargset(throw_ex, 0, jl_builtin_throw); + jl_exprargset(throw_ex, 0, BUILTIN(throw)); jl_exprargset(throw_ex, 1, exc); jl_toplevel_eval_flex(m, throw_ex, 0, 0, &filename, &lineno); JL_GC_POP();