From 14f5c46e972cac3faea65e1c0fd4af49badd099e Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Sat, 8 Feb 2025 03:01:50 +0000 Subject: [PATCH] Minor fixes to support sticky --- src/gc-mmtk.c | 84 +++--------------------------------- src/gc-wb-mmtk.h | 14 ++++++ src/gc-wb-stock.h | 54 +++++++++++++++++++++++ src/julia.h | 108 ++++++---------------------------------------- src/staticdata.c | 6 ++- 5 files changed, 91 insertions(+), 175 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index b40c4492d9abf..5f8524b3a58b9 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -878,6 +878,7 @@ inline void mmtk_set_side_metadata(const void* side_metadata_base, void* obj) { STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { mmtk_set_side_metadata(MMTK_SIDE_LOG_BIT_BASE_ADDRESS, obj); + } } JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int osize, size_t align, void *ty) @@ -1094,6 +1095,11 @@ void jl_gc_notify_image_load(const char* img_data, size_t len) mmtk_set_vm_space((void*)img_data, len); } +void jl_gc_notify_image_alloc(const char* img_data, size_t len) +{ + mmtk_immortal_region_post_alloc((void*)img_data, len); +} + // ========================================================================= // // Code specific to stock that is not supported by MMTk // ========================================================================= // @@ -1143,7 +1149,7 @@ JL_DLLEXPORT void jl_gc_queue_root(const struct _jl_value_t *ptr) JL_NOTSAFEPOIN { jl_task_t *ct = jl_current_task; jl_ptls_t ptls = ct->ptls; - mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, parent, (const void*) 0); + mmtk_object_reference_write_slow(&ptls->gc_tls.mmtk_mutator, ptr, (const void*) 0); } JL_DLLEXPORT void jl_gc_queue_multiroot(const struct _jl_value_t *root, const void *stored, @@ -1225,82 +1231,6 @@ JL_DLLEXPORT jl_value_t *jl_gc_internal_obj_base_ptr(void *p) return NULL; } -JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destdata, - jl_genericmemory_t *src, char* srcdata, - size_t n) JL_NOTSAFEPOINT -{ - jl_datatype_t *dt = (jl_datatype_t*)jl_typetagof(dest); - if (dt != (jl_datatype_t*)jl_typetagof(src)) - jl_exceptionf(jl_argumenterror_type, "jl_genericmemory_copyto requires source and dest to have same type"); - const jl_datatype_layout_t *layout = dt->layout; - if (layout->flags.arrayelem_isboxed) { - _Atomic(void*) * dest_p = (_Atomic(void*)*)destdata; - _Atomic(void*) * src_p = (_Atomic(void*)*)srcdata; - jl_value_t *owner = jl_genericmemory_owner(dest); - jl_gc_wb(owner, NULL); - if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - ssize_t done = 0; - if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) { - if (dest_p < src_p || dest_p > src_p + n) { - for (; done < n; done++) { // copy forwards - void *val = jl_atomic_load_relaxed(src_p + done); - jl_atomic_store_release(dest_p + done, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { - jl_gc_queue_root(owner); - break; - } - } - src_p += done; - dest_p += done; - } else { - for (; done < n; done++) { // copy backwards - void *val = jl_atomic_load_relaxed(src_p + n - done - 1); - jl_atomic_store_release(dest_p + n - done - 1, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) { - jl_gc_queue_root(owner); - break; - } - } - } - n -= done; - } - } - return memmove_refs(dest_p, src_p, n); - } - size_t elsz = layout->size; - char *src_p = srcdata; - int isbitsunion = layout->flags.arrayelem_isunion; - if (isbitsunion) { - char *sourcetypetagdata = jl_genericmemory_typetagdata(src); - char *desttypetagdata = jl_genericmemory_typetagdata(dest); - memmove(desttypetagdata+(size_t)destdata, sourcetypetagdata+(size_t)srcdata, n); - srcdata = (char*)src->ptr + elsz*(size_t)srcdata; - destdata = (char*)dest->ptr + elsz*(size_t)destdata; - } - if (layout->first_ptr != -1) { - memmove_refs((_Atomic(void*)*)destdata, (_Atomic(void*)*)srcdata, n * elsz / sizeof(void*)); - jl_value_t *owner = jl_genericmemory_owner(dest); - if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) { - dt = (jl_datatype_t*)jl_tparam1(dt); - for (size_t done = 0; done < n; done++) { // copy forwards - char* s = (char*)src_p+done*elsz; - if (*((jl_value_t**)s+layout->first_ptr) != NULL) - jl_gc_queue_multiroot(owner, s, dt); - } - } - } - } - else { - memmove(destdata, srcdata, n * elsz); - } -} - - #ifdef __cplusplus } #endif diff --git a/src/gc-wb-mmtk.h b/src/gc-wb-mmtk.h index c20d9ddb6fa32..5652b7a88dd7e 100644 --- a/src/gc-wb-mmtk.h +++ b/src/gc-wb-mmtk.h @@ -13,6 +13,7 @@ extern "C" { extern void mmtk_object_reference_write_post(void* mutator, const void* parent, const void* ptr); extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, const void* ptr); +extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; #define MMTK_OBJECT_BARRIER (1) // Stickyimmix needs write barrier. Immix does not need write barrier. @@ -64,6 +65,19 @@ STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_ mmtk_gc_wb_fast(parent, (void*)0); } +STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, + jl_genericmemory_t *src, _Atomic(void*) * src_p, + size_t* n) JL_NOTSAFEPOINT +{ + mmtk_gc_wb_fast(dest_owner, (void*)0); +} + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, + size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT +{ + mmtk_gc_wb_fast(owner, (void*)0); +} + #ifdef __cplusplus } diff --git a/src/gc-wb-stock.h b/src/gc-wb-stock.h index 2f91dc17c660e..9da402c1c6237 100644 --- a/src/gc-wb-stock.h +++ b/src/gc-wb-stock.h @@ -43,6 +43,60 @@ STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_ jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt); } +STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, + jl_genericmemory_t *src, _Atomic(void*) * src_p, + size_t* n) JL_NOTSAFEPOINT +{ + if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) { + jl_value_t *src_owner = jl_genericmemory_owner(src); + size_t done = 0; + if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { + if (dest_p < src_p || dest_p > src_p + (*n)) { + for (; done < (*n); done++) { // copy forwards + void *val = jl_atomic_load_relaxed(src_p + done); + jl_atomic_store_release(dest_p + done, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { + jl_gc_queue_root(dest_owner); + break; + } + } + src_p += done; + dest_p += done; + } + else { + for (; done < (*n); done++) { // copy backwards + void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1); + jl_atomic_store_release(dest_p + (*n) - done - 1, val); + // `val` is young or old-unmarked + if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { + jl_gc_queue_root(dest_owner); + break; + } + } + } + (*n) -= done; + } + } +} + +STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, + size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT +{ + if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) { + jl_value_t *src_owner = jl_genericmemory_owner(src); + size_t elsz = dt->layout->size; + if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { + dt = (jl_datatype_t*)jl_tparam1(dt); + for (size_t done = 0; done < n; done++) { // copy forwards + char* s = (char*)src_p+done*elsz; + if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL) + jl_gc_queue_multiroot(owner, s, dt); + } + } + } +} + #ifdef __cplusplus } #endif diff --git a/src/julia.h b/src/julia.h index adc180bb21747..304cc340f710c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1177,12 +1177,6 @@ JL_DLLEXPORT void jl_free_stack(void *stkbuf, size_t bufsz); // thread-local allocator of the current thread. JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value); -#ifndef MMTK_GC -#include "gc-wb-stock.h" -#else -#include "gc-wb-mmtk.h" -#endif - JL_DLLEXPORT void jl_gc_safepoint(void); JL_DLLEXPORT int jl_safepoint_suspend_thread(int tid, int waitstate); JL_DLLEXPORT void jl_safepoint_suspend_all_threads(struct _jl_task_t *ct); @@ -1245,6 +1239,18 @@ STATIC_INLINE jl_value_t *jl_svecset( #define jl_array_maxsize(a) (((jl_array_t*)(a))->ref.mem->length) #define jl_array_len(a) (jl_array_ndims(a) == 1 ? jl_array_nrows(a) : jl_array_maxsize(a)) +JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; +#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack)) + +STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; + +// write barriers +#ifndef MMTK_GC +#include "gc-wb-stock.h" +#else +#include "gc-wb-mmtk.h" +#endif + /* how - allocation style 0 = data is inlined @@ -1301,94 +1307,6 @@ STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set( } #endif -// GC write barriers - -STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT -{ - // parent and ptr isa jl_value_t* - if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset - (jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young - jl_gc_queue_root((jl_value_t*)parent); -} - -STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t* -{ - // if ptr is old - if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) { - jl_gc_queue_root((jl_value_t*)ptr); - } -} - -STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT -{ - // 3 == GC_OLD_MARKED - // ptr is an immutable object - if (__likely(jl_astaggedvalue(parent)->bits.gc != 3)) - return; // parent is young or in remset - if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3)) - return; // ptr is old and not in remset (thus it does not point to young) - jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr); - const jl_datatype_layout_t *ly = dt->layout; - if (ly->npointers) - jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt); -} - -STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; - -STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p, - jl_genericmemory_t *src, _Atomic(void*) * src_p, - size_t* n) JL_NOTSAFEPOINT -{ - if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - size_t done = 0; - if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { - if (dest_p < src_p || dest_p > src_p + (*n)) { - for (; done < (*n); done++) { // copy forwards - void *val = jl_atomic_load_relaxed(src_p + done); - jl_atomic_store_release(dest_p + done, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { - jl_gc_queue_root(dest_owner); - break; - } - } - src_p += done; - dest_p += done; - } - else { - for (; done < (*n); done++) { // copy backwards - void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1); - jl_atomic_store_release(dest_p + (*n) - done - 1, val); - // `val` is young or old-unmarked - if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) { - jl_gc_queue_root(dest_owner); - break; - } - } - } - (*n) -= done; - } - } -} - -STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p, - size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT -{ - if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) { - jl_value_t *src_owner = jl_genericmemory_owner(src); - size_t elsz = dt->layout->size; - if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) { - dt = (jl_datatype_t*)jl_tparam1(dt); - for (size_t done = 0; done < n; done++) { // copy forwards - char* s = (char*)src_p+done*elsz; - if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL) - jl_gc_queue_multiroot(owner, s, dt); - } - } - } -} - STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT { jl_genericmemory_t *m_ = (jl_genericmemory_t*)m; @@ -2401,8 +2319,6 @@ JL_DLLEXPORT void JL_NORETURN jl_throw(jl_value_t *e JL_MAYBE_UNROOTED); JL_DLLEXPORT void JL_NORETURN jl_rethrow(void); JL_DLLEXPORT void JL_NORETURN jl_rethrow_other(jl_value_t *e JL_MAYBE_UNROOTED); JL_DLLEXPORT void JL_NORETURN jl_no_exc_handler(jl_value_t *e, jl_task_t *ct); -JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_ROOTED JL_NOTSAFEPOINT; -#define jl_current_task (container_of(jl_get_pgcstack(), jl_task_t, gcstack)) extern JL_DLLIMPORT int jl_task_gcstack_offset; extern JL_DLLIMPORT int jl_task_ptls_offset; diff --git a/src/staticdata.c b/src/staticdata.c index c29448c491eb7..2b8d29876c418 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -4166,9 +4166,10 @@ static jl_value_t *jl_restore_package_image_from_stream(void* pkgimage_handle, i char *sysimg; int success = !needs_permalloc; ios_seek(f, datastartpos); - if (needs_permalloc) + if (needs_permalloc) { sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); - else + jl_gc_notify_image_alloc(sysimg, len); + } else sysimg = &f->buf[f->bpos]; if (needs_permalloc) success = ios_readall(f, sysimg, len) == len; @@ -4291,6 +4292,7 @@ JL_DLLEXPORT void jl_restore_system_image(const char *fname) ios_seek_end(&f); size_t len = ios_pos(&f); char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); + jl_gc_notify_image_alloc(sysimg, len); ios_seek(&f, 0); if (ios_readall(&f, sysimg, len) != len) jl_errorf("Error reading system image file.");