From fe836c81b019f79fe63fe69ed004e222d7687171 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Fri, 22 Sep 2023 21:23:22 -0400 Subject: [PATCH 1/3] Excise Random from the system image --- base/Base.jl | 6 ++++- base/stubs.jl | 44 +++++++++++++++++++++++++++++++++++++ base/sysimg.jl | 2 -- src/jl_exported_funcs.inc | 1 + src/julia.h | 1 + src/module.c | 5 +++++ stdlib/Random/src/Random.jl | 9 ++++++++ stdlib/stdlib.mk | 4 ++-- test/precompile.jl | 38 +++++++++++++++++++------------- 9 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 base/stubs.jl diff --git a/base/Base.jl b/base/Base.jl index 0ca13265adc4f..4d3f24751709c 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -508,6 +508,10 @@ include("docs/basedocs.jl") # Documentation -- should always be included last in sysimg. include("docs/Docs.jl") using .Docs + +# excised stdlib stubs, needs to be after docs +include("stubs.jl") + if isdefined(Core, :Compiler) && is_primary_base_module Docs.loaddocs(Core.Compiler.CoreDocs.DOCS) end @@ -662,7 +666,7 @@ end # enable threads support @eval PCRE PCRE_COMPILE_LOCK = Threads.SpinLock() -end +end # is_primary_base_module # Ensure this file is also tracked @assert !isassigned(_included_files, 1) diff --git a/base/stubs.jl b/base/stubs.jl new file mode 100644 index 0000000000000..a946893be2214 --- /dev/null +++ b/base/stubs.jl @@ -0,0 +1,44 @@ +module Stubs + +module Random + let Random_PkgID = Base.PkgId(Base.UUID(0x9a3f8284_a2c9_5f02_9a11_845980a1fd5c), "Random") + RANDOM_MODULE_REF = Ref{Module}() + + global delay_initialize + function delay_initialize() + if !isassigned(RANDOM_MODULE_REF) + RANDOM_MODULE_REF[] = Base.require(Random_PkgID) + end + return ccall(:jl_module_world, Csize_t, (Any,), RANDOM_MODULE_REF[]) + end + end + + import Base: rand, randn + function rand(args...) + Base.invoke_in_world(delay_initialize(), rand, args...) + end + + function randn(args...) + Base.invoke_in_world(delay_initialize(), randn, args...) + end +end + +Base.Docs.getdoc(::typeof(Base.rand)) = (Random.delay_initialize(); nothing) +Base.Docs.getdoc(::typeof(Base.randn)) = (Random.delay_initialize(); nothing) + +function delete_stubs(mod) + for name in names(mod, imported=true) + if name == :delay_initialize + continue + end + obj = getglobal(mod, name) + if obj isa Function + ms = Base.methods(obj, mod) + for m in ms + Base.delete_method(m) + end + end + end +end + +end diff --git a/base/sysimg.jl b/base/sysimg.jl index 4ae876a929f0e..578bd3b2f35a1 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -73,7 +73,6 @@ let :FileWatching, # used by loading.jl -- implicit assumption that init runs :Libdl, # Transitive through LinAlg :Artifacts, # Transitive through LinAlg - :SHA, # transitive through Random :Sockets, # used by stream.jl # Transitive through LingAlg @@ -82,7 +81,6 @@ let # 1-depth packages :LinearAlgebra, # Commits type-piracy and GEMM - :Random, # Can't be removed due to rand being exported by Base ] # PackageCompiler can filter out stdlibs so it can be empty maxlen = maximum(textwidth.(string.(stdlibs)); init=0) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index f9587d33518a2..a63644f215f4e 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -346,6 +346,7 @@ XX(jl_new_method_table) \ XX(jl_new_method_uninit) \ XX(jl_new_module) \ + XX(jl_module_world) \ XX(jl_new_primitivetype) \ XX(jl_new_struct) \ XX(jl_new_structt) \ diff --git a/src/julia.h b/src/julia.h index 12c920cc6899d..947bc6110ae7c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1890,6 +1890,7 @@ extern JL_DLLIMPORT jl_module_t *jl_base_module JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_module_t *jl_top_module JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_module_t *jl_libdl_module JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name, jl_module_t *parent); +JL_DLLEXPORT size_t jl_module_world(jl_module_t *m); JL_DLLEXPORT void jl_set_module_nospecialize(jl_module_t *self, int on); JL_DLLEXPORT void jl_set_module_optlevel(jl_module_t *self, int lvl); JL_DLLEXPORT int jl_get_module_optlevel(jl_module_t *m); diff --git a/src/module.c b/src/module.c index f44238e45c659..ee26575ebff25 100644 --- a/src/module.c +++ b/src/module.c @@ -63,6 +63,11 @@ uint32_t jl_module_next_counter(jl_module_t *m) return jl_atomic_fetch_add(&m->counter, 1); } +JL_DLLEXPORT size_t jl_module_world(jl_module_t *m) +{ + return m->primary_world; +} + JL_DLLEXPORT jl_value_t *jl_f_new_module(jl_sym_t *name, uint8_t std_imports, uint8_t default_names) { // TODO: should we prohibit this during incremental compilation? diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl index 432e32a4de691..c6039d29b4533 100644 --- a/stdlib/Random/src/Random.jl +++ b/stdlib/Random/src/Random.jl @@ -31,6 +31,15 @@ export rand!, randn!, ## general definitions +module Stubs + function __init__() + # Remove the shim methods + if !Base.generating_output() + Base.Stubs.delete_stubs(Base.Stubs.Random) + end + end +end + """ AbstractRNG diff --git a/stdlib/stdlib.mk b/stdlib/stdlib.mk index 696b24a8f8bf1..dfc5f138c6a7e 100644 --- a/stdlib/stdlib.mk +++ b/stdlib/stdlib.mk @@ -1,11 +1,11 @@ STDLIBS_WITHIN_SYSIMG := \ - Artifacts FileWatching Libdl SHA libblastrampoline_jll OpenBLAS_jll Random \ + Artifacts FileWatching Libdl libblastrampoline_jll OpenBLAS_jll \ LinearAlgebra Sockets INDEPENDENT_STDLIBS := \ ArgTools Base64 CRC32c Dates DelimitedFiles Distributed Downloads Future \ InteractiveUtils LazyArtifacts LibGit2 LibCURL Logging Markdown Mmap \ - NetworkOptions Profile Printf Pkg REPL Serialization SharedArrays SparseArrays \ + NetworkOptions Profile Printf Pkg Random REPL Serialization SHA SharedArrays SparseArrays \ Statistics StyledStrings Tar Test TOML Unicode UUIDs \ dSFMT_jll GMP_jll libLLVM_jll LLD_jll LLVMLibUnwind_jll LibUnwind_jll LibUV_jll \ LibCURL_jll LibSSH2_jll LibGit2_jll nghttp2_jll MozillaCACerts_jll MbedTLS_jll \ diff --git a/test/precompile.jl b/test/precompile.jl index 8a6b81e1aa96e..6726fb3aa2d89 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1074,25 +1074,30 @@ precompile_test_harness("invoke") do dir module $InvokeModule export f, g, h, q, fnc, gnc, hnc, qnc # nc variants do not infer to a Const export f44320, g44320 - export getlast + export getlast, getdata + + # use instead of `rand` + const data = Ref{Float64}(0) + getdata() = data[] + # f is for testing invoke that occurs within a dependency f(x::Real) = 0 f(x::Int) = x < 5 ? 1 : invoke(f, Tuple{Real}, x) - fnc(x::Real) = rand()-1 - fnc(x::Int) = x < 5 ? rand()+1 : invoke(fnc, Tuple{Real}, x) + fnc(x::Real) = getdata()-1 + fnc(x::Int) = x < 5 ? getdata()+1 : invoke(fnc, Tuple{Real}, x) # g is for testing invoke that occurs from a dependent g(x::Real) = 0 g(x::Int) = 1 - gnc(x::Real) = rand()-1 - gnc(x::Int) = rand()+1 + gnc(x::Real) = getdata()-1 + gnc(x::Int) = getdata()+1 # h will be entirely superseded by a new method (full invalidation) h(x::Real) = 0 h(x::Int) = x < 5 ? 1 : invoke(h, Tuple{Integer}, x) - hnc(x::Real) = rand()-1 - hnc(x::Int) = x < 5 ? rand()+1 : invoke(hnc, Tuple{Integer}, x) + hnc(x::Real) = getdata()-1 + hnc(x::Int) = x < 5 ? getdata()+1 : invoke(hnc, Tuple{Integer}, x) # q will have some callers invalidated q(x::Integer) = 0 - qnc(x::Integer) = rand()-1 + qnc(x::Integer) = getdata()-1 # Issue #44320 f44320(::Int) = 1 f44320(::Any) = 2 @@ -1129,8 +1134,8 @@ precompile_test_harness("invoke") do dir # Purely internal internal(x::Real) = 0 internal(x::Int) = x < 5 ? 1 : invoke(internal, Tuple{Real}, x) - internalnc(x::Real) = rand()-1 - internalnc(x::Int) = x < 5 ? rand()+1 : invoke(internalnc, Tuple{Real}, x) + internalnc(x::Real) = getdata()-1 + internalnc(x::Int) = x < 5 ? getdata()+1 : invoke(internalnc, Tuple{Real}, x) # Issue #44320 f44320(::Real) = 3 @@ -1157,10 +1162,10 @@ precompile_test_harness("invoke") do dir # Now that we've precompiled, invalidate with a new method that overrides the `invoke` dispatch $InvokeModule.h(x::Integer) = -1 - $InvokeModule.hnc(x::Integer) = rand() - 20 + $InvokeModule.hnc(x::Integer) = getdata() - 20 # ...and for q, override with a more specialized method that should leave only the invoked version still valid $InvokeModule.q(x::Int) = -1 - $InvokeModule.qnc(x::Int) = rand()+1 + $InvokeModule.qnc(x::Int) = getdata()+1 end """) Base.compilecache(Base.PkgId(string(CallerModule))) @@ -1618,13 +1623,16 @@ precompile_test_harness("No external edges") do load_path write(joinpath(load_path, "NoExternalEdges.jl"), """ module NoExternalEdges - bar(x::Int) = hcat(rand()) - @inline bar() = hcat(rand()) + const data = Ref{Float64}() + getdata() = data[] + + bar(x::Int) = hcat(getdata()) + @inline bar() = hcat(getdata()) bar(x::Float64) = bar() foo1() = bar(1) foo2() = bar(1.0) foo3() = bar() - foo4() = hcat(rand()) + foo4() = hcat(getdata()) precompile(foo1, ()) precompile(foo2, ()) precompile(foo3, ()) From acccf39f022cb6a4180778a45a8be2ffad904859 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 18 Oct 2023 14:39:08 -0400 Subject: [PATCH 2/3] Support deleting methods during precompilation for stdlib excision (#51641) The problem with the `delete_method` in `__init__` approach is that we invalidate the method-table, after we have performed all of the caching work. A package dependent on `Random`, will still see the stub method in Base and thus when we delete the stub, we may invalidate useful work. Instead we delete the methods when Random is being loaded, thus a dependent package only ever sees the method table with all the methods in Random, and non of the stubs methods. The only invalidation that thus may happen are calls to `rand` and `randn` without first doing an `import Random`. --- base/compiler/typeinfer.jl | 1 + base/loading.jl | 1 + base/stubs.jl | 3 ++- doc/src/devdocs/locks.md | 1 + src/clangsa/GCChecker.cpp | 3 ++- src/gf.c | 20 ++++++++++++++++---- src/init.c | 2 ++ src/jl_exported_funcs.inc | 1 + src/julia.h | 7 +++++++ src/staticdata.c | 35 +++++++++++++++++++++++++---------- src/staticdata_utils.c | 34 ++++++++++++++++++++++++++++++++++ stdlib/Random/src/Random.jl | 10 ++-------- 12 files changed, 94 insertions(+), 24 deletions(-) diff --git a/base/compiler/typeinfer.jl b/base/compiler/typeinfer.jl index afae4abc2a07d..f6c4f9cf12215 100644 --- a/base/compiler/typeinfer.jl +++ b/base/compiler/typeinfer.jl @@ -3,6 +3,7 @@ # Tracking of newly-inferred CodeInstances during precompilation const track_newly_inferred = RefValue{Bool}(false) const newly_inferred = CodeInstance[] +const newly_deleted = Method[] # build (and start inferring) the inference frame for the top-level MethodInstance function typeinf(interp::AbstractInterpreter, result::InferenceResult, cache_mode::Symbol) diff --git a/base/loading.jl b/base/loading.jl index 8ea8354031c30..ceb8eff6c1e36 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2414,6 +2414,7 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto end ccall(:jl_set_newly_inferred, Cvoid, (Any,), Core.Compiler.newly_inferred) + ccall(:jl_set_newly_deleted, Cvoid, (Any,), Core.Compiler.newly_deleted) Core.Compiler.track_newly_inferred.x = true try Base.include(Base.__toplevel__, input) diff --git a/base/stubs.jl b/base/stubs.jl index a946893be2214..baaf1f72fef73 100644 --- a/base/stubs.jl +++ b/base/stubs.jl @@ -35,7 +35,8 @@ function delete_stubs(mod) if obj isa Function ms = Base.methods(obj, mod) for m in ms - Base.delete_method(m) + ccall(:jl_push_newly_deleted, Cvoid, (Any,), m) + ccall(:jl_method_table_disable_incremental, Cvoid, (Any, Any), Base.get_methodtable(m), m) end end end diff --git a/doc/src/devdocs/locks.md b/doc/src/devdocs/locks.md index f79f4f2b1e3e3..0295ccc82805e 100644 --- a/doc/src/devdocs/locks.md +++ b/doc/src/devdocs/locks.md @@ -45,6 +45,7 @@ The following is a leaf lock (level 2), and only acquires level 1 locks (safepoi > * Module->lock > * JLDebuginfoPlugin::PluginMutex > * newly_inferred_mutex +> * newly_deleted_mutex The following is a level 3 lock, which can only acquire level 1 or level 2 locks internally: diff --git a/src/clangsa/GCChecker.cpp b/src/clangsa/GCChecker.cpp index f8e73c486a862..97def0fac59f2 100644 --- a/src/clangsa/GCChecker.cpp +++ b/src/clangsa/GCChecker.cpp @@ -1444,7 +1444,8 @@ bool GCChecker::evalCall(const CallEvent &Call, CheckerContext &C) const { } else if (name == "JL_GC_PUSH1" || name == "JL_GC_PUSH2" || name == "JL_GC_PUSH3" || name == "JL_GC_PUSH4" || name == "JL_GC_PUSH5" || name == "JL_GC_PUSH6" || - name == "JL_GC_PUSH7" || name == "JL_GC_PUSH8") { + name == "JL_GC_PUSH7" || name == "JL_GC_PUSH8" || + name == "JL_GC_PUSH9") { ProgramStateRef State = C.getState(); // Transform slots to roots, transform values to rooted unsigned NumArgs = CE->getNumArgs(); diff --git a/src/gf.c b/src/gf.c index 3c5870487ec9b..e7f59b98af126 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1872,9 +1872,9 @@ static jl_typemap_entry_t *do_typemap_search(jl_methtable_t *mt JL_PROPAGATES_RO } #endif -static void jl_method_table_invalidate(jl_methtable_t *mt, jl_typemap_entry_t *methodentry, size_t max_world) +static void jl_method_table_invalidate(jl_methtable_t *mt, jl_typemap_entry_t *methodentry, size_t max_world, int tracked) { - if (jl_options.incremental && jl_generating_output()) + if (!tracked && jl_options.incremental && jl_generating_output()) jl_error("Method deletion is not possible during Module precompile."); jl_method_t *method = methodentry->func.method; assert(!method->is_for_opaque_closure); @@ -1931,10 +1931,22 @@ JL_DLLEXPORT void jl_method_table_disable(jl_methtable_t *mt, jl_method_t *metho JL_LOCK(&mt->writelock); // Narrow the world age on the method to make it uncallable size_t world = jl_atomic_fetch_add(&jl_world_counter, 1); - jl_method_table_invalidate(mt, methodentry, world); + jl_method_table_invalidate(mt, methodentry, world, 0); JL_UNLOCK(&mt->writelock); } +JL_DLLEXPORT void jl_method_table_disable_incremental(jl_methtable_t *mt, jl_method_t *method) +{ + jl_typemap_entry_t *methodentry = do_typemap_search(mt, method); + JL_LOCK(&mt->writelock); + // Narrow the world age on the method to make it uncallable + // size_t world = jl_atomic_load_acquire(&jl_world_counter); + size_t world = jl_atomic_fetch_add(&jl_world_counter, 1); + jl_method_table_invalidate(mt, methodentry, world, 1); + JL_UNLOCK(&mt->writelock); +} + + static int jl_type_intersection2(jl_value_t *t1, jl_value_t *t2, jl_value_t **isect JL_REQUIRE_ROOTED_SLOT, jl_value_t **isect2 JL_REQUIRE_ROOTED_SLOT) { *isect2 = NULL; @@ -2025,7 +2037,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method oldvalue = (jl_value_t*)replaced; invalidated = 1; method_overwrite(newentry, replaced->func.method); - jl_method_table_invalidate(mt, replaced, max_world); + jl_method_table_invalidate(mt, replaced, max_world, 0); } else { jl_method_t *const *d; diff --git a/src/init.c b/src/init.c index d4128c8ae9e40..ecea16f03a279 100644 --- a/src/init.c +++ b/src/init.c @@ -718,6 +718,7 @@ static void jl_set_io_wait(int v) extern jl_mutex_t jl_modules_mutex; extern jl_mutex_t precomp_statement_out_lock; extern jl_mutex_t newly_inferred_mutex; +extern jl_mutex_t newly_deleted_mutex; extern jl_mutex_t global_roots_lock; extern jl_mutex_t profile_show_peek_cond_lock; @@ -736,6 +737,7 @@ static void init_global_mutexes(void) { JL_MUTEX_INIT(&jl_modules_mutex, "jl_modules_mutex"); JL_MUTEX_INIT(&precomp_statement_out_lock, "precomp_statement_out_lock"); JL_MUTEX_INIT(&newly_inferred_mutex, "newly_inferred_mutex"); + JL_MUTEX_INIT(&newly_deleted_mutex, "newly_deleted_mutex"); JL_MUTEX_INIT(&global_roots_lock, "global_roots_lock"); JL_MUTEX_INIT(&jl_codegen_lock, "jl_codegen_lock"); JL_MUTEX_INIT(&typecache_lock, "typecache_lock"); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index a63644f215f4e..c1e83f2fbda98 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -317,6 +317,7 @@ XX(jl_method_instance_add_backedge) \ XX(jl_method_table_add_backedge) \ XX(jl_method_table_disable) \ + XX(jl_method_table_disable_incremental) \ XX(jl_method_table_for) \ XX(jl_method_table_insert) \ XX(jl_methtable_lookup) \ diff --git a/src/julia.h b/src/julia.h index 947bc6110ae7c..51af20797bc3f 100644 --- a/src/julia.h +++ b/src/julia.h @@ -958,6 +958,7 @@ extern void JL_GC_PUSH5(void *, void *, void *, void *, void *) JL_NOTSAFEPOINT extern void JL_GC_PUSH6(void *, void *, void *, void *, void *, void *) JL_NOTSAFEPOINT; extern void JL_GC_PUSH7(void *, void *, void *, void *, void *, void *, void *) JL_NOTSAFEPOINT; extern void JL_GC_PUSH8(void *, void *, void *, void *, void *, void *, void *, void *) JL_NOTSAFEPOINT; +extern void JL_GC_PUSH9(void *, void *, void *, void *, void *, void *, void *, void *, void *) JL_NOTSAFEPOINT; extern void _JL_GC_PUSHARGS(jl_value_t **, size_t) JL_NOTSAFEPOINT; // This is necessary, because otherwise the analyzer considers this undefined // behavior and terminates the exploration @@ -1000,6 +1001,9 @@ extern void JL_GC_POP() JL_NOTSAFEPOINT; #define JL_GC_PUSH8(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) \ void *__gc_stkf[] = {(void*)JL_GC_ENCODE_PUSH(8), jl_pgcstack, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8}; \ jl_pgcstack = (jl_gcframe_t*)__gc_stkf; +#define JL_GC_PUSH9(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) \ + void *__gc_stkf[] = {(void*)JL_GC_ENCODE_PUSH(9), jl_pgcstack, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9}; \ + jl_pgcstack = (jl_gcframe_t*)__gc_stkf; #define JL_GC_PUSHARGS(rts_var,n) \ @@ -2053,6 +2057,9 @@ JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *d JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t *newly_inferred); JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t *ci); +JL_DLLEXPORT void jl_method_table_disable_incremental(jl_methtable_t *mt, jl_method_t *m); +JL_DLLEXPORT void jl_set_newly_deleted(jl_value_t *newly_deleted); +JL_DLLEXPORT void jl_push_newly_deleted(jl_value_t *m); JL_DLLEXPORT void jl_write_compiler_output(void); // parsing diff --git a/src/staticdata.c b/src/staticdata.c index e8ac99043b78d..e9ea5b48d5a6d 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -2566,7 +2566,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_array_t *worklist, jl_array_t *extext_methods, jl_array_t *new_ext_cis, jl_array_t *method_roots_list, - jl_array_t *ext_targets, jl_array_t *edges) JL_GC_DISABLED + jl_array_t *ext_targets, jl_array_t *edges, jl_array_t *newly_deleted) JL_GC_DISABLED { htable_new(&field_replace, 0); // strip metadata and IR when requested @@ -2688,6 +2688,10 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_queue_for_serialization(&s, ext_targets); jl_queue_for_serialization(&s, edges); } + if (newly_deleted) { + jl_queue_for_serialization(&s, newly_deleted); + } + jl_serialize_reachable(&s); // step 1.2: ensure all gvars are part of the sysimage too record_gvars(&s, &gvars); @@ -2841,6 +2845,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_write_value(&s, method_roots_list); jl_write_value(&s, ext_targets); jl_write_value(&s, edges); + jl_write_value(&s, newly_deleted); } write_uint32(f, jl_array_len(s.link_ids_gctags)); ios_write(f, (char*)jl_array_data(s.link_ids_gctags, uint32_t), jl_array_len(s.link_ids_gctags) * sizeof(uint32_t)); @@ -2926,11 +2931,11 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_ext_cis = NULL; - jl_array_t *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; + jl_array_t *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL, *_newly_deleted = NULL; int64_t checksumpos = 0; int64_t checksumpos_ff = 0; int64_t datastartpos = 0; - JL_GC_PUSH6(&mod_array, &extext_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges); + JL_GC_PUSH7(&mod_array, &extext_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges, &_newly_deleted); if (worklist) { mod_array = jl_get_loaded_modules(); // __toplevel__ modules loaded in this session (from Base.loaded_modules_array) @@ -2977,7 +2982,10 @@ JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *workli } if (_native_data != NULL) native_functions = *_native_data; - jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, method_roots_list, ext_targets, edges); + // Otherwise serialization will be confused. + if (newly_deleted) + _newly_deleted = jl_array_copy(newly_deleted); + jl_save_system_image_to_stream(ff, mod_array, worklist, extext_methods, new_ext_cis, method_roots_list, ext_targets, edges, _newly_deleted); if (_native_data != NULL) native_functions = NULL; // make sure we don't run any Julia code concurrently before this point @@ -3061,6 +3069,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl jl_array_t **extext_methods, jl_array_t **new_ext_cis, jl_array_t **method_roots_list, jl_array_t **ext_targets, jl_array_t **edges, + jl_array_t **newly_deleted, char **base, arraylist_t *ccallable_list, pkgcachesizes *cachesizes) JL_GC_DISABLED { int en = jl_gc_enable(0); @@ -3122,7 +3131,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl assert(!ios_eof(f)); s.s = f; uintptr_t offset_restored = 0, offset_init_order = 0, offset_extext_methods = 0, offset_new_ext_cis = 0, offset_method_roots_list = 0; - uintptr_t offset_ext_targets = 0, offset_edges = 0; + uintptr_t offset_ext_targets = 0, offset_edges = 0, offset_newly_deleted = 0; if (!s.incremental) { size_t i; for (i = 0; tags[i] != NULL; i++) { @@ -3157,6 +3166,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl offset_method_roots_list = jl_read_offset(&s); offset_ext_targets = jl_read_offset(&s); offset_edges = jl_read_offset(&s); + offset_newly_deleted = jl_read_offset(&s); } s.buildid_depmods_idxs = depmod_to_imageidx(depmods); size_t nlinks_gctags = read_uint32(f); @@ -3190,8 +3200,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl *method_roots_list = (jl_array_t*)jl_delayed_reloc(&s, offset_method_roots_list); *ext_targets = (jl_array_t*)jl_delayed_reloc(&s, offset_ext_targets); *edges = (jl_array_t*)jl_delayed_reloc(&s, offset_edges); + *newly_deleted = (jl_array_t*)jl_delayed_reloc(&s, offset_newly_deleted); } - s.s = NULL; // step 3: apply relocations assert(!ios_eof(f)); @@ -3378,6 +3388,11 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl assert(jl_is_datatype(obj)); jl_cache_type_((jl_datatype_t*)obj); } + + // Delete methods before inserting new ones. + if (newly_deleted) + jl_delete_methods(*newly_deleted); + // Perform fixups: things like updating world ages, inserting methods & specializations, etc. size_t world = jl_atomic_load_acquire(&jl_world_counter); for (size_t i = 0; i < s.uniquing_objs.len; i++) { @@ -3598,11 +3613,11 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i assert(datastartpos > 0 && datastartpos < dataendpos); needs_permalloc = jl_options.permalloc_pkgimg || needs_permalloc; jl_value_t *restored = NULL; - jl_array_t *init_order = NULL, *extext_methods = NULL, *new_ext_cis = NULL, *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; + jl_array_t *init_order = NULL, *extext_methods = NULL, *new_ext_cis = NULL, *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL, *newly_deleted = NULL; jl_svec_t *cachesizes_sv = NULL; char *base; arraylist_t ccallable_list; - JL_GC_PUSH8(&restored, &init_order, &extext_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges, &cachesizes_sv); + JL_GC_PUSH9(&restored, &init_order, &extext_methods, &new_ext_cis, &method_roots_list, &ext_targets, &edges, &newly_deleted, &cachesizes_sv); { // make a permanent in-memory copy of f (excluding the header) ios_bufmode(f, bm_none); @@ -3627,7 +3642,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i ios_static_buffer(f, sysimg, len); pkgcachesizes cachesizes; jl_restore_system_image_from_stream_(f, image, depmods, checksum, (jl_array_t**)&restored, &init_order, &extext_methods, &new_ext_cis, &method_roots_list, - &ext_targets, &edges, &base, &ccallable_list, &cachesizes); + &ext_targets, &edges, &newly_deleted, &base, &ccallable_list, &cachesizes); JL_SIGATOMIC_END(); // Insert method extensions @@ -3668,7 +3683,7 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i static void jl_restore_system_image_from_stream(ios_t *f, jl_image_t *image, uint32_t checksum) { JL_TIMING(LOAD_IMAGE, LOAD_Sysimg); - jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + jl_restore_system_image_from_stream_(f, image, NULL, checksum | ((uint64_t)0xfdfcfbfa << 32), NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } JL_DLLEXPORT jl_value_t *jl_restore_incremental_from_buf(void* pkgimage_handle, const char *buf, jl_image_t *image, size_t sz, jl_array_t *depmods, int completeinfo, const char *pkgname, int needs_permalloc) diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 982f4b104eb74..36984d2282f9e 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -103,6 +103,27 @@ JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t* ci) JL_UNLOCK(&newly_inferred_mutex); } +static jl_array_t *newly_deleted JL_GLOBALLY_ROOTED /*FIXME*/; +// Mutex for newly_deleted +jl_mutex_t newly_deleted_mutex; + +// Register array of newly-inferred MethodInstances +// This gets called as the first step of Base.include_package_for_output +JL_DLLEXPORT void jl_set_newly_deleted(jl_value_t* _newly_deleted) +{ + assert(newly_deleted == NULL || jl_is_array(_newly_deleted)); + newly_deleted = (jl_array_t*) _newly_deleted; +} + +JL_DLLEXPORT void jl_push_newly_deleted(jl_value_t* m) +{ + JL_LOCK(&newly_deleted_mutex); + size_t end = jl_array_len(newly_deleted); + jl_array_grow_end(newly_deleted, 1); + jl_array_ptr_set(newly_deleted, end, m); + JL_UNLOCK(&newly_deleted_mutex); +} + // compute whether a type references something internal to worklist // and thus could not have existed before deserialize @@ -841,6 +862,19 @@ static void jl_insert_methods(jl_array_t *list) } } +static void jl_delete_methods(jl_array_t *list) +{ + size_t i, l = jl_array_len(list); + for (i = 0; i < l; i++) { + jl_method_t *meth = (jl_method_t*)jl_array_ptr_ref(list, i); + assert(jl_is_method(meth)); + assert(!meth->is_for_opaque_closure); + jl_methtable_t *mt = jl_method_get_table(meth); + assert((jl_value_t*)mt != jl_nothing); + jl_method_table_disable_incremental(mt, meth); + } +} + static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) { size_t i, l = jl_array_nrows(method_roots_list); diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl index c6039d29b4533..ae43a0d21188f 100644 --- a/stdlib/Random/src/Random.jl +++ b/stdlib/Random/src/Random.jl @@ -31,14 +31,8 @@ export rand!, randn!, ## general definitions -module Stubs - function __init__() - # Remove the shim methods - if !Base.generating_output() - Base.Stubs.delete_stubs(Base.Stubs.Random) - end - end -end +# Remove the shim methods +Base.Stubs.delete_stubs(Base.Stubs.Random) """ AbstractRNG From 923efb95e798295b36a20485b1bd876e0d22631b Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Mon, 4 Dec 2023 16:08:12 -0500 Subject: [PATCH 3/3] Give a deprecation warning --- base/Base.jl | 4 ---- base/deprecated.jl | 5 +++++ base/exports.jl | 4 ---- base/stubs.jl | 29 +++++++++++++++++------------ test/gc/chunks.jl | 1 + test/testhelpers/coverage_file.jl | 1 + 6 files changed, 24 insertions(+), 20 deletions(-) diff --git a/base/Base.jl b/base/Base.jl index 4d3f24751709c..1d5a497df5987 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -377,10 +377,6 @@ using .CoreLogging include("env.jl") -# functions defined in Random -function rand end -function randn end - # I/O include("libuv.jl") include("asyncevent.jl") diff --git a/base/deprecated.jl b/base/deprecated.jl index 7e7fa5596ca25..9e698346af25e 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -396,4 +396,9 @@ end @deprecate permute!!(a, p::AbstractVector{<:Integer}) permute!(a, p) false @deprecate invpermute!!(a, p::AbstractVector{<:Integer}) invpermute!(a, p) false +# functions defined in Random +function rand end +function randn end +export rand, randn + # END 1.11 deprecations diff --git a/base/exports.jl b/base/exports.jl index 92525b85c7635..4265e0ef1e91a 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -995,10 +995,6 @@ export unsafe_store!, unsafe_swap!, -# implemented in Random module - rand, - randn, - # Macros # parser internal @__FILE__, diff --git a/base/stubs.jl b/base/stubs.jl index baaf1f72fef73..f3c74ddda7e08 100644 --- a/base/stubs.jl +++ b/base/stubs.jl @@ -1,6 +1,21 @@ module Stubs +macro stub(old, mod) + dep_message = """: `$old` has been moved to the standard library package `$mod`. + Add `using $mod` to your imports.""" + return Expr(:toplevel, + quote + import Base: $old + function $(esc(old))(args...) + Base.depwarn($dep_message, $(QuoteNode(old))) + Base.invoke_in_world($(esc(:delay_initialize))(), $(esc(old)), args...) + end + Base.Docs.getdoc(::typeof($(esc(old)))) = ($(esc(:delay_initialize))(); nothing) + end) +end + module Random + import ..Stubs: @stub let Random_PkgID = Base.PkgId(Base.UUID(0x9a3f8284_a2c9_5f02_9a11_845980a1fd5c), "Random") RANDOM_MODULE_REF = Ref{Module}() @@ -12,20 +27,10 @@ module Random return ccall(:jl_module_world, Csize_t, (Any,), RANDOM_MODULE_REF[]) end end - - import Base: rand, randn - function rand(args...) - Base.invoke_in_world(delay_initialize(), rand, args...) - end - - function randn(args...) - Base.invoke_in_world(delay_initialize(), randn, args...) - end + @stub rand Random + @stub randn Random end -Base.Docs.getdoc(::typeof(Base.rand)) = (Random.delay_initialize(); nothing) -Base.Docs.getdoc(::typeof(Base.randn)) = (Random.delay_initialize(); nothing) - function delete_stubs(mod) for name in names(mod, imported=true) if name == :delay_initialize diff --git a/test/gc/chunks.jl b/test/gc/chunks.jl index 08af59ecbf973..db0ce43ef0963 100644 --- a/test/gc/chunks.jl +++ b/test/gc/chunks.jl @@ -1,4 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +using Random # MWE from https://github.com/JuliaLang/julia/issues/49501 N = 1_000_000 # or larger diff --git a/test/testhelpers/coverage_file.jl b/test/testhelpers/coverage_file.jl index e8e0355952d80..557243b096ec3 100644 --- a/test/testhelpers/coverage_file.jl +++ b/test/testhelpers/coverage_file.jl @@ -1,4 +1,5 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +using Random function code_coverage_test() if rand(1:2) == 3