Skip to content

Commit

Permalink
Moving write barrier away from genericmemory.c
Browse files Browse the repository at this point in the history
  • Loading branch information
@udesou
udesou committed Feb 20, 2025
1 parent 0fb5fa0 commit 0647635
Show file tree
Hide file tree
Showing 3 changed files with 106 additions and 74 deletions.
15 changes: 15 additions & 0 deletions src/gc-interface.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#define JL_GC_INTERFACE_H

#include "dtypes.h"
#include "julia_atomics.h"

#ifdef __cplusplus
extern "C" {
Expand All @@ -17,6 +18,7 @@ struct _jl_tls_states_t;
struct _jl_value_t;
struct _jl_weakref_t;
struct _jl_datatype_t;
struct _jl_genericmemory_t;

// ========================================================================= //
// GC Metrics
Expand Down Expand Up @@ -253,6 +255,19 @@ STATIC_INLINE void jl_gc_wb_knownold(const void *parent, const void *ptr) JL_NOT
STATIC_INLINE void jl_gc_multi_wb(const void *parent,
const struct _jl_value_t *ptr) JL_NOTSAFEPOINT;

// Write-barrier function that must be used after copying fields of elements of genericmemory objects
// into another. It should be semantically equivalent to triggering multiple write barriers – one
// per field of the object being copied, but may be special-cased for performance reasons.
STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const struct _jl_value_t *owner, struct _jl_genericmemory_t *src, char* src_p,
size_t n, struct _jl_datatype_t *dt) JL_NOTSAFEPOINT;

// Similar to jl_gc_wb_genericmemory_copy but must be used when copying *boxed* elements of a genericmemory
// object. Note that this barrier also performs the copying unlike jl_gc_wb_genericmemory_copy_ptr.
// The parameters src_p, dest_p and n will be modified and will contain information about
// the *uncopied* data after performing this barrier, and will be copied using memmove_refs.
STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const struct _jl_value_t *owner, _Atomic(void*) * dest_p,
struct _jl_genericmemory_t *src, _Atomic(void*) * src_p,
size_t* n) JL_NOTSAFEPOINT;
#ifdef __cplusplus
}
#endif
Expand Down
43 changes: 2 additions & 41 deletions src/genericmemory.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -235,36 +235,7 @@ JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destda
_Atomic(void*) * dest_p = (_Atomic(void*)*)destdata;
_Atomic(void*) * src_p = (_Atomic(void*)*)srcdata;
jl_value_t *owner = jl_genericmemory_owner(dest);
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
ssize_t done = 0;
if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) {
if (dest_p < src_p || dest_p > src_p + n) {
for (; done < n; done++) { // copy forwards
void *val = jl_atomic_load_relaxed(src_p + done);
jl_atomic_store_release(dest_p + done, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
jl_gc_queue_root(owner);
break;
}
}
src_p += done;
dest_p += done;
} else {
for (; done < n; done++) { // copy backwards
void *val = jl_atomic_load_relaxed(src_p + n - done - 1);
jl_atomic_store_release(dest_p + n - done - 1, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
jl_gc_queue_root(owner);
break;
}
}
}
n -= done;
}
}
jl_gc_wb_genericmemory_copy_boxed(owner, dest_p, src, src_p, &n);
return memmove_refs(dest_p, src_p, n);
}
size_t elsz = layout->size;
Expand All @@ -280,17 +251,7 @@ JL_DLLEXPORT void jl_genericmemory_copyto(jl_genericmemory_t *dest, char* destda
if (layout->first_ptr != -1) {
memmove_refs((_Atomic(void*)*)destdata, (_Atomic(void*)*)srcdata, n * elsz / sizeof(void*));
jl_value_t *owner = jl_genericmemory_owner(dest);
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
if (jl_astaggedvalue(src_owner)->bits.gc != GC_OLD_MARKED) {
dt = (jl_datatype_t*)jl_tparam1(dt);
for (size_t done = 0; done < n; done++) { // copy forwards
char* s = (char*)src_p+done*elsz;
if (*((jl_value_t**)s+layout->first_ptr) != NULL)
jl_gc_queue_multiroot(owner, s, dt);
}
}
}
jl_gc_wb_genericmemory_copy_ptr(owner, src, src_p, n, dt);
}
else {
memmove(destdata, srcdata, n * elsz);
Expand Down
122 changes: 89 additions & 33 deletions src/julia.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ typedef struct {
// jl_value_t *data[];
} jl_svec_t;

JL_EXTENSION typedef struct {
JL_EXTENSION typedef struct _jl_genericmemory_t {
JL_DATA_TYPE
size_t length;
void *ptr;
Expand Down Expand Up @@ -1182,38 +1182,6 @@ JL_DLLEXPORT void jl_free_stack(void *stkbuf, size_t bufsz);
// thread-local allocator of the current thread.
JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value);

// GC write barriers

STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT
{
// parent and ptr isa jl_value_t*
if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset
(jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young
jl_gc_queue_root((jl_value_t*)parent);
}

STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t*
{
// if ptr is old
if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) {
jl_gc_queue_root((jl_value_t*)ptr);
}
}

STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT
{
// 3 == GC_OLD_MARKED
// ptr is an immutable object
if (__likely(jl_astaggedvalue(parent)->bits.gc != 3))
return; // parent is young or in remset
if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3))
return; // ptr is old and not in remset (thus it does not point to young)
jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr);
const jl_datatype_layout_t *ly = dt->layout;
if (ly->npointers)
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
}

JL_DLLEXPORT void jl_gc_safepoint(void);
JL_DLLEXPORT int jl_safepoint_suspend_thread(int tid, int waitstate);
JL_DLLEXPORT void jl_safepoint_suspend_all_threads(struct _jl_task_t *ct);
Expand Down Expand Up @@ -1332,6 +1300,94 @@ STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set(
}
#endif

// GC write barriers

STATIC_INLINE void jl_gc_wb(const void *parent, const void *ptr) JL_NOTSAFEPOINT
{
// parent and ptr isa jl_value_t*
if (__unlikely(jl_astaggedvalue(parent)->bits.gc == 3 /* GC_OLD_MARKED */ && // parent is old and not in remset
(jl_astaggedvalue(ptr)->bits.gc & 1 /* GC_MARKED */) == 0)) // ptr is young
jl_gc_queue_root((jl_value_t*)parent);
}

STATIC_INLINE void jl_gc_wb_back(const void *ptr) JL_NOTSAFEPOINT // ptr isa jl_value_t*
{
// if ptr is old
if (__unlikely(jl_astaggedvalue(ptr)->bits.gc == 3 /* GC_OLD_MARKED */)) {
jl_gc_queue_root((jl_value_t*)ptr);
}
}

STATIC_INLINE void jl_gc_multi_wb(const void *parent, const jl_value_t *ptr) JL_NOTSAFEPOINT
{
// 3 == GC_OLD_MARKED
// ptr is an immutable object
if (__likely(jl_astaggedvalue(parent)->bits.gc != 3))
return; // parent is young or in remset
if (__likely(jl_astaggedvalue(ptr)->bits.gc == 3))
return; // ptr is old and not in remset (thus it does not point to young)
jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(ptr);
const jl_datatype_layout_t *ly = dt->layout;
if (ly->npointers)
jl_gc_queue_multiroot((jl_value_t*)parent, ptr, dt);
}

STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;

STATIC_INLINE void jl_gc_wb_genericmemory_copy_boxed(const jl_value_t *dest_owner, _Atomic(void*) * dest_p,
jl_genericmemory_t *src, _Atomic(void*) * src_p,
size_t* n) JL_NOTSAFEPOINT
{
if (__unlikely(jl_astaggedvalue(dest_owner)->bits.gc == 3 /* GC_OLD_MARKED */ )) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
size_t done = 0;
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
if (dest_p < src_p || dest_p > src_p + (*n)) {
for (; done < (*n); done++) { // copy forwards
void *val = jl_atomic_load_relaxed(src_p + done);
jl_atomic_store_release(dest_p + done, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
jl_gc_queue_root(dest_owner);
break;
}
}
src_p += done;
dest_p += done;
}
else {
for (; done < (*n); done++) { // copy backwards
void *val = jl_atomic_load_relaxed(src_p + (*n) - done - 1);
jl_atomic_store_release(dest_p + (*n) - done - 1, val);
// `val` is young or old-unmarked
if (val && !(jl_astaggedvalue(val)->bits.gc & 1 /* GC_MARKED */)) {
jl_gc_queue_root(dest_owner);
break;
}
}
}
(*n) -= done;
}
}
}

STATIC_INLINE void jl_gc_wb_genericmemory_copy_ptr(const jl_value_t *owner, jl_genericmemory_t *src, char* src_p,
size_t n, jl_datatype_t *dt) JL_NOTSAFEPOINT
{
if (__unlikely(jl_astaggedvalue(owner)->bits.gc == 3 /* GC_OLD_MARKED */)) {
jl_value_t *src_owner = jl_genericmemory_owner(src);
size_t elsz = dt->layout->size;
if (jl_astaggedvalue(src_owner)->bits.gc != 3 /* GC_OLD_MARKED */) {
dt = (jl_datatype_t*)jl_tparam1(dt);
for (size_t done = 0; done < n; done++) { // copy forwards
char* s = (char*)src_p+done*elsz;
if (*((jl_value_t**)s+dt->layout->first_ptr) != NULL)
jl_gc_queue_multiroot(owner, s, dt);
}
}
}
}

STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT
{
jl_genericmemory_t *m_ = (jl_genericmemory_t*)m;
Expand Down

0 comments on commit 0647635

Please sign in to comment.