Merge branch 'master' into test_compiler_codegen_ci

base/client.jl

@@ -265,7 +265,7 @@ function exec_options(opts)
     distributed_mode = (opts.worker == 1) || (opts.nprocs > 0) || (opts.machine_file != C_NULL)
     if distributed_mode
         let Distributed = require(PkgId(UUID((0x8ba89e20_285c_5b6f, 0x9357_94700520ee1b)), "Distributed"))
-            Core.eval(MainInclude, :(const Distributed = $Distributed))
+            MainInclude.Distributed = Distributed
             Core.eval(Main, :(using Base.MainInclude.Distributed))
             invokelatest(Distributed.process_opts, opts)
         end
@@ -400,7 +400,7 @@ function load_InteractiveUtils(mod::Module=Main)
         try
             # TODO: we have to use require_stdlib here because it is a dependency of REPL, but we would sort of prefer not to
             let InteractiveUtils = require_stdlib(PkgId(UUID(0xb77e0a4c_d291_57a0_90e8_8db25a27a240), "InteractiveUtils"))
-                Core.eval(MainInclude, :(const InteractiveUtils = $InteractiveUtils))
+                MainInclude.InteractiveUtils = InteractiveUtils
             end
         catch ex
             @warn "Failed to import InteractiveUtils into module $mod" exception=(ex, catch_backtrace())
@@ -535,6 +535,10 @@ The thrown errors are collected in a stack of exceptions.
 """
 global err = nothing
 
+# Used for memoizing require_stdlib of these modules
+global InteractiveUtils::Module
+global Distributed::Module
+
 # weakly exposes ans and err variables to Main
 export ans, err
 end

base/idset.jl

@@ -4,7 +4,7 @@
     IdSet{T}([itr])
     IdSet()
 
-IdSet{T}() constructs a set (see [`Set`](@ref)) using
+`IdSet{T}()` constructs a set (see [`Set`](@ref)) using
 `===` as equality with values of type `T`.
 
 In the example below, the values are all `isequal` so they get overwritten in the ordinary `Set`.

base/invalidation.jl

@@ -180,26 +180,34 @@ function binding_was_invalidated(b::Core.Binding)
     b.partitions.min_world > unsafe_load(cglobal(:jl_require_world, UInt))
 end
 
-function scan_new_method!(methods_with_invalidated_source::IdSet{Method}, method::Method)
+function scan_new_method!(methods_with_invalidated_source::IdSet{Method}, method::Method, image_backedges_only::Bool)
     isdefined(method, :source) || return
+    if image_backedges_only && !has_image_globalref(method)
+        return
+    end
     src = _uncompressed_ir(method)
     mod = method.module
     foreachgr(src) do gr::GlobalRef
         b = convert(Core.Binding, gr)
-        binding_was_invalidated(b) && push!(methods_with_invalidated_source, method)
+        if binding_was_invalidated(b)
+            # TODO: We could turn this into an addition if condition. For now, use it as a reasonably cheap
+            # additional consistency chekc
+            @assert !image_backedges_only
+            push!(methods_with_invalidated_source, method)
+        end
         maybe_add_binding_backedge!(b, method)
     end
 end
 
-function scan_new_methods(extext_methods::Vector{Any}, internal_methods::Vector{Any})
+function scan_new_methods(extext_methods::Vector{Any}, internal_methods::Vector{Any}, image_backedges_only::Bool)
     methods_with_invalidated_source = IdSet{Method}()
     for method in internal_methods
         if isa(method, Method)
-           scan_new_method!(methods_with_invalidated_source, method)
+           scan_new_method!(methods_with_invalidated_source, method, image_backedges_only)
         end
     end
     for tme::Core.TypeMapEntry in extext_methods
-        scan_new_method!(methods_with_invalidated_source, tme.func::Method)
+        scan_new_method!(methods_with_invalidated_source, tme.func::Method, image_backedges_only)
     end
     return methods_with_invalidated_source
 end

base/runtime_internals.jl

@@ -1656,3 +1656,5 @@ isempty(mt::Core.MethodTable) = (mt.defs === nothing)
 uncompressed_ir(m::Method) = isdefined(m, :source) ? _uncompressed_ir(m) :
                              isdefined(m, :generator) ? error("Method is @generated; try `code_lowered` instead.") :
                              error("Code for this Method is not available.")
+
+has_image_globalref(m::Method) = ccall(:jl_ir_flag_has_image_globalref, Bool, (Any,), m.source)

base/staticdata.jl

@@ -25,7 +25,8 @@ end
 function insert_backedges(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{Any}}, extext_methods::Vector{Any}, internal_methods::Vector{Any})
     # determine which CodeInstance objects are still valid in our image
     # to enable any applicable new codes
-    methods_with_invalidated_source = Base.scan_new_methods(extext_methods, internal_methods)
+    backedges_only = unsafe_load(cglobal(:jl_first_image_replacement_world, UInt)) == typemax(UInt)
+    methods_with_invalidated_source = Base.scan_new_methods(extext_methods, internal_methods, backedges_only)
     stack = CodeInstance[]
     visiting = IdDict{CodeInstance,Int}()
     _insert_backedges(edges, stack, visiting, methods_with_invalidated_source)

src/ircode.c

@@ -547,14 +547,15 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal)
     }
 }
 
-static jl_code_info_flags_t code_info_flags(uint8_t propagate_inbounds, uint8_t has_fcall,
+static jl_code_info_flags_t code_info_flags(uint8_t propagate_inbounds, uint8_t has_fcall, uint8_t has_image_globalref,
                                             uint8_t nospecializeinfer, uint8_t isva,
                                             uint8_t inlining, uint8_t constprop, uint8_t nargsmatchesmethod,
                                             jl_array_t *ssaflags)
 {
     jl_code_info_flags_t flags;
     flags.bits.propagate_inbounds = propagate_inbounds;
     flags.bits.has_fcall = has_fcall;
+    flags.bits.has_image_globalref = has_image_globalref;
     flags.bits.nospecializeinfer = nospecializeinfer;
     flags.bits.isva = isva;
     flags.bits.inlining = inlining;
@@ -1036,7 +1037,7 @@ JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code)
     };
 
     uint8_t nargsmatchesmethod = code->nargs == m->nargs;
-    jl_code_info_flags_t flags = code_info_flags(code->propagate_inbounds, code->has_fcall,
+    jl_code_info_flags_t flags = code_info_flags(code->propagate_inbounds, code->has_fcall, code->has_image_globalref,
                                                  code->nospecializeinfer, code->isva,
                                                  code->inlining, code->constprop,
                                                  nargsmatchesmethod,
@@ -1134,6 +1135,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t
     code->constprop = flags.bits.constprop;
     code->propagate_inbounds = flags.bits.propagate_inbounds;
     code->has_fcall = flags.bits.has_fcall;
+    code->has_image_globalref = flags.bits.has_image_globalref;
     code->nospecializeinfer = flags.bits.nospecializeinfer;
     code->isva = flags.bits.isva;
     code->purity.bits = read_uint16(s.s);
@@ -1228,6 +1230,16 @@ JL_DLLEXPORT uint8_t jl_ir_flag_has_fcall(jl_string_t *data)
     return flags.bits.has_fcall;
 }
 
+JL_DLLEXPORT uint8_t jl_ir_flag_has_image_globalref(jl_string_t *data)
+{
+    if (jl_is_code_info(data))
+        return ((jl_code_info_t*)data)->has_image_globalref;
+    assert(jl_is_string(data));
+    jl_code_info_flags_t flags;
+    flags.packed = jl_string_data(data)[ir_offset_flags];
+    return flags.bits.has_image_globalref;
+}
+
 JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_string_t *data)
 {
     if (jl_is_code_info(data))

src/jltypes.c

@@ -3485,7 +3485,7 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_code_info_type =
         jl_new_datatype(jl_symbol("CodeInfo"), core,
                         jl_any_type, jl_emptysvec,
-                        jl_perm_symsvec(22,
+                        jl_perm_symsvec(23,
                             "code",
                             "debuginfo",
                             "ssavaluetypes",
@@ -3502,13 +3502,14 @@ void jl_init_types(void) JL_GC_DISABLED
                             "nargs",
                             "propagate_inbounds",
                             "has_fcall",
+                            "has_image_globalref",
                             "nospecializeinfer",
                             "isva",
                             "inlining",
                             "constprop",
                             "purity",
                             "inlining_cost"),
-                        jl_svec(22,
+                        jl_svec(23,
                             jl_array_any_type,
                             jl_debuginfo_type,
                             jl_any_type,
@@ -3527,6 +3528,7 @@ void jl_init_types(void) JL_GC_DISABLED
                             jl_bool_type,
                             jl_bool_type,
                             jl_bool_type,
+                            jl_bool_type,
                             jl_uint8_type,
                             jl_uint8_type,
                             jl_uint16_type,

src/julia.h

@@ -312,6 +312,7 @@ typedef struct _jl_code_info_t {
     // various boolean properties:
     uint8_t propagate_inbounds;
     uint8_t has_fcall;
+    uint8_t has_image_globalref;
     uint8_t nospecializeinfer;
     uint8_t isva;
     // uint8 settings
@@ -2263,6 +2264,7 @@ JL_DLLEXPORT jl_value_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code);
 JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t *metadata, jl_value_t *data);
 JL_DLLEXPORT uint8_t jl_ir_flag_inlining(jl_value_t *data) JL_NOTSAFEPOINT;
 JL_DLLEXPORT uint8_t jl_ir_flag_has_fcall(jl_value_t *data) JL_NOTSAFEPOINT;
+JL_DLLEXPORT uint8_t jl_ir_flag_has_image_globalref(jl_value_t *data) JL_NOTSAFEPOINT;
 JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_value_t *data) JL_NOTSAFEPOINT;
 JL_DLLEXPORT ssize_t jl_ir_nslots(jl_value_t *data) JL_NOTSAFEPOINT;
 JL_DLLEXPORT uint8_t jl_ir_slotflag(jl_value_t *data, size_t i) JL_NOTSAFEPOINT;

src/julia_internal.h

@@ -650,6 +650,7 @@ STATIC_INLINE jl_value_t *undefref_check(jl_datatype_t *dt, jl_value_t *v) JL_NO
 typedef struct {
     uint16_t propagate_inbounds:1;
     uint16_t has_fcall:1;
+    uint16_t has_image_globalref:1;
     uint16_t nospecializeinfer:1;
     uint16_t isva:1;
     uint16_t nargsmatchesmethod:1;

src/llvm-final-gc-lowering.cpp

@@ -161,21 +161,26 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
     target->replaceAllUsesWith(newI);
     target->eraseFromParent();
 }
+bool FinalLowerGC::shouldRunFinalGC(Function &F)
+{
+    bool should_run = 0;
+    should_run |= getOrNull(jl_intrinsics ::newGCFrame) != nullptr;
+    should_run |= getOrNull(jl_intrinsics ::getGCFrameSlot) != nullptr;
+    should_run |= getOrNull(jl_intrinsics ::pushGCFrame) != nullptr;
+    should_run |= getOrNull(jl_intrinsics ::popGCFrame) != nullptr;
+    should_run |= getOrNull(jl_intrinsics ::GCAllocBytes) != nullptr;
+    should_run |= getOrNull(jl_intrinsics ::queueGCRoot) != nullptr;
+    should_run |= getOrNull(jl_intrinsics ::safepoint) != nullptr;
+    return should_run;
+}
 
 bool FinalLowerGC::runOnFunction(Function &F)
 {
     initAll(*F.getParent());
-    if (!pgcstack_getter && !adoptthread_func) {
-        LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Skipping function " << F.getName() << "\n");
-        return false;
-    }
-
-    // Look for a call to 'julia.get_pgcstack'.
     pgcstack = getPGCstack(F);
-    if (!pgcstack) {
-        LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Skipping function " << F.getName() << " no pgcstack\n");
-        return false;
-    }
+    if (!shouldRunFinalGC(F))
+        goto verify_skip;
+
     LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Processing function " << F.getName() << "\n");
     queueRootFunc = getOrDeclare(jl_well_known::GCQueueRoot);
     smallAllocFunc = getOrDeclare(jl_well_known::GCSmallAlloc);
@@ -212,8 +217,37 @@ bool FinalLowerGC::runOnFunction(Function &F)
 #undef LOWER_INTRINSIC
         }
     }
-
     return true;
+    // Verify that skipping was in fact correct
+    verify_skip:
+    #ifdef JL_VERIFY_PASSES
+        for (auto &BB : F) {
+            for (auto &I : make_early_inc_range(BB)) {
+                auto *CI = dyn_cast<CallInst>(&I);
+                if (!CI)
+                    continue;
+
+            Value *callee = CI->getCalledOperand();
+            assert(callee);
+            auto IS_INTRINSIC = [&](auto intrinsic) {
+                auto intrinsic2 = getOrNull(intrinsic);
+                if (intrinsic2 == callee) {
+                    errs() << "Final-GC-lowering didn't eliminate all intrinsics'" << F.getName() << "', dumping entire module!\n\n";
+                    errs() << *F.getParent() << "\n";
+                    abort();
+                }
+            };
+            IS_INTRINSIC(jl_intrinsics::newGCFrame);
+            IS_INTRINSIC(jl_intrinsics::pushGCFrame);
+            IS_INTRINSIC(jl_intrinsics::popGCFrame);
+            IS_INTRINSIC(jl_intrinsics::getGCFrameSlot);
+            IS_INTRINSIC(jl_intrinsics::GCAllocBytes);
+            IS_INTRINSIC(jl_intrinsics::queueGCRoot);
+            IS_INTRINSIC(jl_intrinsics::safepoint);
+            }
+        }
+    #endif
+    return false;
 }
 
 PreservedAnalyses FinalLowerGCPass::run(Function &F, FunctionAnalysisManager &AM)

src/llvm-gc-interface-passes.h

@@ -411,6 +411,8 @@ struct FinalLowerGC: private JuliaPassContext {
 
     // Lowers a `julia.safepoint` intrinsic.
     void lowerSafepoint(CallInst *target, Function &F);
+    // Check if the pass should be run
+    bool shouldRunFinalGC(Function &F);
 };
 
 #endif // LLVM_GC_PASSES_H