From 47ce04386deb702e5cc751f159bc1d02e3f7693d Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Sat, 30 Dec 2017 17:52:45 +0100 Subject: [PATCH 01/10] Increase GC root field to 32-bit Previously the GC root was stored in the zend_refcounted header in 14 bits, with an additional 2 bits used for the GC color. This limits us to a root buffer of at most ~16k entries, which is no longer tenable. This commit performs the following changes: * Arrays and objects now use the zend_refcounted_gc header, which is zend_refcounted followed by a u32 gc_root field. * The field in zend_refcounted that previously held the gc_info is now extra_flags. This flag space is freely usable for type-specific flags. * The GC color is now stored as part of the GC flags. * The object flags have been moved from the GC flags to the extra flags, as there is no longer enough space for them. TODO: The array flags need to be moved into extra flags as well, to make sure that these changes are size-neutral on x64. --- Zend/zend_gc.c | 83 ++++++++++++++++---------------------- Zend/zend_gc.h | 28 ++----------- Zend/zend_hash.c | 9 +++-- Zend/zend_objects.c | 1 + Zend/zend_objects_API.h | 4 +- Zend/zend_types.h | 89 +++++++++++++++++++++++++++-------------- 6 files changed, 104 insertions(+), 110 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index 39ed6a96a980d..a32acb27e1a4d 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -104,16 +104,21 @@ ZEND_API int (*gc_collect_cycles)(void); # define GC_TRACE(str) #endif -#define GC_REF_SET_ADDRESS(ref, a) \ - GC_INFO_SET_ADDRESS(GC_INFO(ref), a) #define GC_REF_GET_COLOR(ref) \ - GC_INFO_GET_COLOR(GC_INFO(ref)) + (GC_FLAGS(ref) & GC_COLOR) +#define GC_REF_SET_COLOR_EX(ref, c) \ + do { GC_FLAGS(ref) = (GC_FLAGS(ref) & ~GC_COLOR) | (c); } while (0); +#define GC_REF_SET_BLACK_EX(ref) \ + do { GC_FLAGS(ref) = GC_FLAGS(ref) & ~GC_COLOR; } while (0); +#define GC_REF_SET_PURPLE_EX(ref) \ + do { GC_FLAGS(ref) |= GC_COLOR; } while (0); + #define GC_REF_SET_COLOR(ref, c) \ - do { GC_TRACE_SET_COLOR(ref, c); GC_INFO_SET_COLOR(GC_INFO(ref), c); } while (0) + do { GC_TRACE_SET_COLOR(ref, c); GC_REF_SET_COLOR_EX(ref, c); } while (0) #define GC_REF_SET_BLACK(ref) \ - do { GC_TRACE_SET_COLOR(ref, GC_BLACK); GC_INFO_SET_BLACK(GC_INFO(ref)); } while (0) + do { GC_TRACE_SET_COLOR(ref, GC_BLACK); GC_REF_SET_BLACK_EX(ref); } while (0) #define GC_REF_SET_PURPLE(ref) \ - do { GC_TRACE_SET_COLOR(ref, GC_PURPLE); GC_INFO_SET_PURPLE(GC_INFO(ref)); } while (0) + do { GC_TRACE_SET_COLOR(ref, GC_PURPLE); GC_REF_SET_PURPLE_EX(ref); } while (0) #if ZEND_GC_DEBUG > 1 static const char *gc_color_name(uint32_t color) { @@ -129,18 +134,18 @@ static void gc_trace_ref(zend_refcounted *ref) { if (GC_TYPE(ref) == IS_OBJECT) { zend_object *obj = (zend_object *) ref; fprintf(stderr, "[%p] rc=%d addr=%d %s object(%s)#%d ", - ref, GC_REFCOUNT(ref), GC_ADDRESS(GC_INFO(ref)), + ref, GC_REFCOUNT(ref), GC_ADDRESS(ref), gc_color_name(GC_REF_GET_COLOR(ref)), obj->ce->name->val, obj->handle); } else if (GC_TYPE(ref) == IS_ARRAY) { zend_array *arr = (zend_array *) ref; fprintf(stderr, "[%p] rc=%d addr=%d %s array(%d) ", - ref, GC_REFCOUNT(ref), GC_ADDRESS(GC_INFO(ref)), + ref, GC_REFCOUNT(ref), GC_ADDRESS(ref), gc_color_name(GC_REF_GET_COLOR(ref)), zend_hash_num_elements(arr)); } else { fprintf(stderr, "[%p] rc=%d addr=%d %s %s ", - ref, GC_REFCOUNT(ref), GC_ADDRESS(GC_INFO(ref)), + ref, GC_REFCOUNT(ref), GC_ADDRESS(ref), gc_color_name(GC_REF_GET_COLOR(ref)), zend_get_type_by_const(GC_TYPE(ref))); } @@ -268,7 +273,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT); ZEND_ASSERT(EXPECTED(GC_REF_GET_COLOR(ref) == GC_BLACK)); - ZEND_ASSERT(!GC_ADDRESS(GC_INFO(ref))); + ZEND_ASSERT(!GC_ADDRESS(ref)); GC_BENCH_INC(zval_possible_root); @@ -289,7 +294,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) zval_dtor_func(ref); return; } - if (UNEXPECTED(GC_INFO(ref))) { + if (UNEXPECTED(GC_ADDRESS(ref))) { return; } newRoot = GC_G(unused); @@ -306,7 +311,8 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) } GC_TRACE_SET_COLOR(ref, GC_PURPLE); - GC_INFO(ref) = (newRoot - GC_G(buf)) | GC_PURPLE; + GC_SET_ADDRESS(ref, newRoot - GC_G(buf)); + GC_REF_SET_PURPLE(ref); newRoot->ref = ref; newRoot->next = GC_G(roots).next; @@ -325,7 +331,7 @@ static zend_always_inline gc_root_buffer* gc_find_additional_buffer(zend_refcoun /* We have to check each additional_buffer to find which one holds the ref */ while (additional_buffer) { - uint32_t idx = GC_ADDRESS(GC_INFO(ref)) - GC_ROOT_BUFFER_MAX_ENTRIES; + uint32_t idx = GC_ADDRESS(ref) - GC_ROOT_BUFFER_MAX_ENTRIES; if (idx < additional_buffer->used) { gc_root_buffer *root = additional_buffer->buf + idx; if (root->ref == ref) { @@ -343,12 +349,12 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) { gc_root_buffer *root; - ZEND_ASSERT(GC_ADDRESS(GC_INFO(ref))); + ZEND_ASSERT(GC_ADDRESS(ref)); GC_BENCH_INC(zval_remove_from_buffer); - if (EXPECTED(GC_ADDRESS(GC_INFO(ref)) < GC_ROOT_BUFFER_MAX_ENTRIES)) { - root = GC_G(buf) + GC_ADDRESS(GC_INFO(ref)); + if (EXPECTED(GC_ADDRESS(ref) < GC_ROOT_BUFFER_MAX_ENTRIES)) { + root = GC_G(buf) + GC_ADDRESS(ref); gc_remove_from_roots(root); } else { root = gc_find_additional_buffer(ref); @@ -357,7 +363,7 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) if (GC_REF_GET_COLOR(ref) != GC_BLACK) { GC_TRACE_SET_COLOR(ref, GC_PURPLE); } - GC_INFO(ref) = 0; + GC_SET_ADDRESS(ref, 0); /* updete next root that is going to be freed */ if (GC_G(next_to_free) == root) { @@ -699,21 +705,13 @@ static void gc_add_garbage(zend_refcounted *ref) if (buf) { GC_G(unused) = buf->prev; -#if 1 /* optimization: color is already GC_BLACK (0) */ - GC_INFO(ref) = buf - GC_G(buf); -#else - GC_REF_SET_ADDRESS(ref, buf - GC_G(buf)); -#endif + GC_SET_ADDRESS(ref, buf - GC_G(buf)); } else if (GC_G(first_unused) != GC_G(last_unused)) { buf = GC_G(first_unused); GC_G(first_unused)++; -#if 1 /* optimization: color is already GC_BLACK (0) */ - GC_INFO(ref) = buf - GC_G(buf); -#else - GC_REF_SET_ADDRESS(ref, buf - GC_G(buf)); -#endif + GC_SET_ADDRESS(ref, buf - GC_G(buf)); } else { /* If we don't have free slots in the buffer, allocate a new one and * set it's address above GC_ROOT_BUFFER_MAX_ENTRIES that have special @@ -726,12 +724,8 @@ static void gc_add_garbage(zend_refcounted *ref) GC_G(additional_buffer) = new_buffer; } buf = GC_G(additional_buffer)->buf + GC_G(additional_buffer)->used; -#if 1 /* optimization: color is already GC_BLACK (0) */ - GC_INFO(ref) = GC_ROOT_BUFFER_MAX_ENTRIES + GC_G(additional_buffer)->used; -#else - GC_REF_SET_ADDRESS(ref, GC_ROOT_BUFFER_MAX_ENTRIES) + GC_G(additional_buffer)->used; -#endif + GC_SET_ADDRESS(ref, GC_ROOT_BUFFER_MAX_ENTRIES + GC_G(additional_buffer)->used); GC_G(additional_buffer)->used++; } if (buf) { @@ -770,12 +764,7 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags) zval *zv, *end; zval tmp; -#if 1 - /* optimization: color is GC_BLACK (0) */ - if (!GC_INFO(ref)) { -#else - if (!GC_ADDRESS(GC_INFO(ref))) { -#endif + if (!GC_ADDRESS(ref)) { gc_add_garbage(ref); } if (obj->handlers->dtor_obj && @@ -816,12 +805,7 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags) return count; } } else if (GC_TYPE(ref) == IS_ARRAY) { -#if 1 - /* optimization: color is GC_BLACK (0) */ - if (!GC_INFO(ref)) { -#else - if (!GC_ADDRESS(GC_INFO(ref))) { -#endif + if (!GC_ADDRESS(ref)) { gc_add_garbage(ref); } ht = (zend_array*)ref; @@ -889,12 +873,12 @@ static int gc_collect_roots(uint32_t *flags) while (current != &GC_G(roots)) { gc_root_buffer *next = current->next; if (GC_REF_GET_COLOR(current->ref) == GC_BLACK) { - if (EXPECTED(GC_ADDRESS(GC_INFO(current->ref)) < GC_ROOT_BUFFER_MAX_ENTRIES)) { + if (EXPECTED(GC_ADDRESS(current->ref) < GC_ROOT_BUFFER_MAX_ENTRIES)) { gc_remove_from_roots(current); } else { gc_remove_from_additional_roots(current); } - GC_INFO(current->ref) = 0; /* reset GC_ADDRESS() and keep GC_BLACK */ + GC_SET_ADDRESS(current->ref, 0); /* reset GC_ADDRESS() and keep GC_BLACK */ } current = next; } @@ -937,16 +921,17 @@ static void gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buff tail_call: if (root || - (GC_ADDRESS(GC_INFO(ref)) != 0 && + ((GC_FLAGS(ref) & GC_COLLECTABLE) && + GC_ADDRESS(ref) != 0 && GC_REF_GET_COLOR(ref) == GC_BLACK)) { GC_TRACE_REF(ref, "removing from buffer"); if (root) { - if (EXPECTED(GC_ADDRESS(GC_INFO(root->ref)) < GC_ROOT_BUFFER_MAX_ENTRIES)) { + if (EXPECTED(GC_ADDRESS(root->ref) < GC_ROOT_BUFFER_MAX_ENTRIES)) { gc_remove_from_roots(root); } else { gc_remove_from_additional_roots(root); } - GC_INFO(ref) = 0; + GC_SET_ADDRESS(ref, 0); root = NULL; } else { GC_REMOVE_FROM_BUFFER(ref); diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index f0bc5610cdbbd..bec464124dfc0 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -40,26 +40,6 @@ # define GC_BENCH_PEAK(peak, counter) #endif -#define GC_COLOR 0xc000 - -#define GC_BLACK 0x0000 -#define GC_WHITE 0x8000 -#define GC_GREY 0x4000 -#define GC_PURPLE 0xc000 - -#define GC_ADDRESS(v) \ - ((v) & ~GC_COLOR) -#define GC_INFO_GET_COLOR(v) \ - (((zend_uintptr_t)(v)) & GC_COLOR) -#define GC_INFO_SET_ADDRESS(v, a) \ - do {(v) = ((v) & GC_COLOR) | (a);} while (0) -#define GC_INFO_SET_COLOR(v, c) \ - do {(v) = ((v) & ~GC_COLOR) | (c);} while (0) -#define GC_INFO_SET_BLACK(v) \ - do {(v) = (v) & ~GC_COLOR;} while (0) -#define GC_INFO_SET_PURPLE(v) \ - do {(v) = (v) | GC_COLOR;} while (0) - typedef struct _gc_root_buffer { zend_refcounted *ref; struct _gc_root_buffer *next; /* double-linked list */ @@ -133,16 +113,14 @@ ZEND_API int zend_gc_collect_cycles(void); END_EXTERN_C() #define GC_REMOVE_FROM_BUFFER(p) do { \ - zend_refcounted *_p = (zend_refcounted*)(p); \ - if (GC_ADDRESS(GC_INFO(_p))) { \ + zend_refcounted *_p = (zend_refcounted *)(p); \ + if (GC_ADDRESS(_p)) { \ gc_remove_from_buffer(_p); \ } \ } while (0) #define GC_MAY_LEAK(ref) \ - ((GC_TYPE_INFO(ref) & \ - (GC_INFO_MASK | (GC_COLLECTABLE << GC_FLAGS_SHIFT))) == \ - (GC_COLLECTABLE << GC_FLAGS_SHIFT)) + ((GC_FLAGS(ref) & GC_COLLECTABLE) && GC_ADDRESS(ref) == 0) static zend_always_inline void gc_check_possible_root(zend_refcounted *ref) { diff --git a/Zend/zend_hash.c b/Zend/zend_hash.c index f5670dd3709fa..0e3155a6d1fa9 100644 --- a/Zend/zend_hash.c +++ b/Zend/zend_hash.c @@ -158,8 +158,9 @@ static const uint32_t uninitialized_bucket[-HT_MIN_MASK] = {HT_INVALID_IDX, HT_INVALID_IDX}; ZEND_API const HashTable zend_empty_array = { - .gc.refcount = 2, - .gc.u.type_info = IS_ARRAY | (GC_IMMUTABLE << GC_FLAGS_SHIFT), + .gc.rc.refcount = 2, + .gc.rc.u.type_info = IS_ARRAY | (GC_IMMUTABLE << GC_FLAGS_SHIFT), + .gc.gc_root = 0, .u.flags = HASH_FLAG_STATIC_KEYS, .nTableMask = HT_MIN_MASK, .arData = (Bucket*)&uninitialized_bucket[2], @@ -175,6 +176,7 @@ static zend_always_inline void _zend_hash_init_int(HashTable *ht, uint32_t nSize { GC_SET_REFCOUNT(ht, 1); GC_TYPE_INFO(ht) = IS_ARRAY | (persistent ? (GC_PERSISTENT << GC_FLAGS_SHIFT) : (GC_COLLECTABLE << GC_FLAGS_SHIFT)); + ht->gc.gc_root = 0; HT_FLAGS(ht) = HASH_FLAG_STATIC_KEYS; ht->nTableMask = HT_MIN_MASK; HT_SET_DATA_ADDR(ht, &uninitialized_bucket); @@ -1362,7 +1364,7 @@ ZEND_API void ZEND_FASTCALL zend_array_destroy(HashTable *ht) /* break possible cycles */ GC_REMOVE_FROM_BUFFER(ht); - GC_TYPE_INFO(ht) = IS_NULL | (GC_WHITE << 16); + GC_TYPE_INFO(ht) = IS_NULL | (GC_WHITE << GC_FLAGS_SHIFT); if (ht->nNumUsed) { /* In some rare cases destructors of regular arrays may be changed */ @@ -1828,6 +1830,7 @@ ZEND_API HashTable* ZEND_FASTCALL zend_array_dup(HashTable *source) ALLOC_HASHTABLE(target); GC_SET_REFCOUNT(target, 1); GC_TYPE_INFO(target) = IS_ARRAY | (GC_COLLECTABLE << GC_FLAGS_SHIFT); + GC_SET_ADDRESS(target, 0); target->nTableSize = source->nTableSize; target->pDestructor = source->pDestructor; diff --git a/Zend/zend_objects.c b/Zend/zend_objects.c index 1e362e678df5a..4ee7d354f41ce 100644 --- a/Zend/zend_objects.c +++ b/Zend/zend_objects.c @@ -31,6 +31,7 @@ ZEND_API void ZEND_FASTCALL zend_object_std_init(zend_object *object, zend_class { GC_SET_REFCOUNT(object, 1); GC_TYPE_INFO(object) = IS_OBJECT | (GC_COLLECTABLE << GC_FLAGS_SHIFT); + object->gc.gc_root = 0; object->ce = ce; object->properties = NULL; zend_objects_store_put(object); diff --git a/Zend/zend_objects_API.h b/Zend/zend_objects_API.h index a581c4b2090f7..a040cf11a5b05 100644 --- a/Zend/zend_objects_API.h +++ b/Zend/zend_objects_API.h @@ -74,8 +74,8 @@ static zend_always_inline void zend_object_release(zend_object *obj) { if (GC_DELREF(obj) == 0) { zend_objects_store_del(obj); - } else if (UNEXPECTED(GC_MAY_LEAK((zend_refcounted*)obj))) { - gc_possible_root((zend_refcounted*)obj); + } else if (UNEXPECTED(GC_MAY_LEAK(obj))) { + gc_possible_root((zend_refcounted *) obj); } } diff --git a/Zend/zend_types.h b/Zend/zend_types.h index a284eb2293379..9dc3068ee5afc 100644 --- a/Zend/zend_types.h +++ b/Zend/zend_types.h @@ -204,19 +204,30 @@ struct _zval_struct { } u2; }; -typedef struct _zend_refcounted_h { +typedef struct _zend_refcounted_common { uint32_t refcount; /* reference counter 32-bit */ union { struct { ZEND_ENDIAN_LOHI_3( zend_uchar type, - zend_uchar flags, /* used for strings & objects */ - uint16_t gc_info) /* keeps GC root number (or 0) and color */ + zend_uchar gc_flags, /* GC flags, including GC color */ + uint16_t extra_flags) /* additional type specific flags */ } v; uint32_t type_info; } u; +} zend_refcounted_common; + +/* This wrapper exists so that refcounted_h and refcounted_gc_h use the same + * field nesting to access the common RC information. */ +typedef struct _zend_refcounted_h { + zend_refcounted_common rc; } zend_refcounted_h; +typedef struct _zend_refcounted_gc_h { + zend_refcounted_common rc; + uint32_t gc_root; +} zend_refcounted_gc_h; + struct _zend_refcounted { zend_refcounted_h gc; }; @@ -237,7 +248,7 @@ typedef struct _Bucket { typedef struct _zend_array HashTable; struct _zend_array { - zend_refcounted_h gc; + zend_refcounted_gc_h gc; union { struct { ZEND_ENDIAN_LOHI_4( @@ -336,7 +347,7 @@ typedef struct _HashTableIterator { } HashTableIterator; struct _zend_object { - zend_refcounted_h gc; + zend_refcounted_gc_h gc; uint32_t handle; // TODO: may be removed ??? zend_class_entry *ce; const zend_object_handlers *handlers; @@ -442,17 +453,20 @@ static zend_always_inline zend_uchar zval_get_type(const zval* pz) { #define Z_TYPE_FLAGS_SHIFT 8 -#define GC_REFCOUNT(p) zend_gc_refcount(&(p)->gc) -#define GC_SET_REFCOUNT(p, rc) zend_gc_set_refcount(&(p)->gc, rc) -#define GC_ADDREF(p) zend_gc_addref(&(p)->gc) -#define GC_DELREF(p) zend_gc_delref(&(p)->gc) -#define GC_ADDREF_EX(p, rc) zend_gc_addref_ex(&(p)->gc, rc) -#define GC_DELREF_EX(p, rc) zend_gc_delref_ex(&(p)->gc, rc) +#define GC_REFCOUNT(p) zend_gc_refcount(&(p)->gc.rc) +#define GC_SET_REFCOUNT(p, rcnt) zend_gc_set_refcount(&(p)->gc.rc, rcnt) +#define GC_ADDREF(p) zend_gc_addref(&(p)->gc.rc) +#define GC_DELREF(p) zend_gc_delref(&(p)->gc.rc) +#define GC_ADDREF_EX(p, rcnt) zend_gc_addref_ex(&(p)->gc.rc, rcnt) +#define GC_DELREF_EX(p, rcnt) zend_gc_delref_ex(&(p)->gc.rc, rcnt) -#define GC_TYPE(p) (p)->gc.u.v.type -#define GC_FLAGS(p) (p)->gc.u.v.flags -#define GC_INFO(p) (p)->gc.u.v.gc_info -#define GC_TYPE_INFO(p) (p)->gc.u.type_info +#define GC_TYPE(p) (p)->gc.rc.u.v.type +#define GC_FLAGS(p) (p)->gc.rc.u.v.gc_flags +#define GC_EXTRA_FLAGS(p) (p)->gc.rc.u.v.extra_flags +#define GC_TYPE_INFO(p) (p)->gc.rc.u.type_info + +#define GC_ADDRESS(p) zend_gc_address(&(p)->gc.rc) +#define GC_SET_ADDRESS(p, addr) zend_gc_set_address(&(p)->gc.rc, addr) #define Z_GC_TYPE(zval) GC_TYPE(Z_COUNTED(zval)) #define Z_GC_TYPE_P(zval_p) Z_GC_TYPE(*(zval_p)) @@ -460,22 +474,25 @@ static zend_always_inline zend_uchar zval_get_type(const zval* pz) { #define Z_GC_FLAGS(zval) GC_FLAGS(Z_COUNTED(zval)) #define Z_GC_FLAGS_P(zval_p) Z_GC_FLAGS(*(zval_p)) -#define Z_GC_INFO(zval) GC_INFO(Z_COUNTED(zval)) -#define Z_GC_INFO_P(zval_p) Z_GC_INFO(*(zval_p)) #define Z_GC_TYPE_INFO(zval) GC_TYPE_INFO(Z_COUNTED(zval)) #define Z_GC_TYPE_INFO_P(zval_p) Z_GC_TYPE_INFO(*(zval_p)) #define GC_FLAGS_SHIFT 8 -#define GC_INFO_SHIFT 16 -#define GC_INFO_MASK 0xffff0000 -/* zval.value->gc.u.v.flags (common flags) */ +/* zval.value->gc.rc.u.v.gc_flags (common flags) */ #define GC_COLLECTABLE (1<<0) #define GC_PROTECTED (1<<1) /* used for recursion detection */ #define GC_IMMUTABLE (1<<2) /* can't be canged in place */ #define GC_PERSISTENT (1<<3) /* allocated using malloc */ #define GC_PERSISTENT_LOCAL (1<<4) /* persistent, but thread-local */ +/* GC colors */ +#define GC_COLOR (3<<6) +#define GC_BLACK (0<<6) +#define GC_WHITE (1<<6) +#define GC_GREY (2<<6) +#define GC_PURPLE (3<<6) + #define GC_ARRAY (IS_ARRAY | (GC_COLLECTABLE << GC_FLAGS_SHIFT)) #define GC_OBJECT (IS_OBJECT | (GC_COLLECTABLE << GC_FLAGS_SHIFT)) @@ -493,7 +510,7 @@ static zend_always_inline zend_uchar zval_get_type(const zval* pz) { #define IS_CONSTANT_AST_EX (IS_CONSTANT_AST | (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT)) -/* string flags (zval.value->gc.u.flags) */ +/* string flags (zval.value->gc.rc.u.gc_flags) */ #define IS_STR_INTERNED GC_IMMUTABLE /* interned string */ #define IS_STR_PERSISTENT GC_PERSISTENT /* allocated using malloc */ #define IS_STR_PERMANENT (1<<5) /* relives request boundary */ @@ -502,13 +519,13 @@ static zend_always_inline zend_uchar zval_get_type(const zval* pz) { #define IS_ARRAY_IMMUTABLE GC_IMMUTABLE #define IS_ARRAY_PERSISTENT GC_PERSISTENT -/* object flags (zval.value->gc.u.flags) */ +/* object flags (zval.value->gc.rc.u.extra_flags) */ #define IS_OBJ_DESTRUCTOR_CALLED (1<<4) #define IS_OBJ_FREE_CALLED (1<<5) #define IS_OBJ_USE_GUARDS (1<<6) #define IS_OBJ_HAS_GUARDS (1<<7) -#define OBJ_FLAGS(obj) GC_FLAGS(obj) +#define OBJ_FLAGS(obj) GC_EXTRA_FLAGS(obj) /* Recursion protection macros must be used only for arrays and objects */ #define GC_IS_RECURSIVE(p) \ @@ -902,8 +919,8 @@ static zend_always_inline zend_uchar zval_get_type(const zval* pz) { extern ZEND_API zend_bool zend_rc_debug; # define ZEND_RC_MOD_CHECK(p) do { \ if (zend_rc_debug) { \ - ZEND_ASSERT(!((p)->u.v.flags & GC_IMMUTABLE)); \ - ZEND_ASSERT(((p)->u.v.flags & (GC_PERSISTENT|GC_PERSISTENT_LOCAL)) != GC_PERSISTENT); \ + ZEND_ASSERT(!((p)->u.v.gc_flags & GC_IMMUTABLE)); \ + ZEND_ASSERT(((p)->u.v.gc_flags & (GC_PERSISTENT|GC_PERSISTENT_LOCAL)) != GC_PERSISTENT); \ } \ } while (0) # define GC_MAKE_PERSISTENT_LOCAL(p) do { \ @@ -916,37 +933,47 @@ extern ZEND_API zend_bool zend_rc_debug; do { } while (0) #endif -static zend_always_inline uint32_t zend_gc_refcount(const zend_refcounted_h *p) { +static zend_always_inline uint32_t zend_gc_refcount(const zend_refcounted_common *p) { return p->refcount; } -static zend_always_inline uint32_t zend_gc_set_refcount(zend_refcounted_h *p, uint32_t rc) { +static zend_always_inline uint32_t zend_gc_set_refcount(zend_refcounted_common *p, uint32_t rc) { p->refcount = rc; return p->refcount; } -static zend_always_inline uint32_t zend_gc_addref(zend_refcounted_h *p) { +static zend_always_inline uint32_t zend_gc_addref(zend_refcounted_common *p) { ZEND_RC_MOD_CHECK(p); return ++(p->refcount); } -static zend_always_inline uint32_t zend_gc_delref(zend_refcounted_h *p) { +static zend_always_inline uint32_t zend_gc_delref(zend_refcounted_common *p) { ZEND_RC_MOD_CHECK(p); return --(p->refcount); } -static zend_always_inline uint32_t zend_gc_addref_ex(zend_refcounted_h *p, uint32_t rc) { +static zend_always_inline uint32_t zend_gc_addref_ex(zend_refcounted_common *p, uint32_t rc) { ZEND_RC_MOD_CHECK(p); p->refcount += rc; return p->refcount; } -static zend_always_inline uint32_t zend_gc_delref_ex(zend_refcounted_h *p, uint32_t rc) { +static zend_always_inline uint32_t zend_gc_delref_ex(zend_refcounted_common *p, uint32_t rc) { ZEND_RC_MOD_CHECK(p); p->refcount -= rc; return p->refcount; } +static zend_always_inline uint32_t zend_gc_address(zend_refcounted_common *p) { + ZEND_ASSERT(p->u.v.gc_flags & GC_COLLECTABLE); + return ((zend_refcounted_gc_h *) p)->gc_root; +} + +static zend_always_inline void zend_gc_set_address(zend_refcounted_common *p, uint32_t addr) { + ZEND_ASSERT(p->u.v.gc_flags & GC_COLLECTABLE); + ((zend_refcounted_gc_h *) p)->gc_root = addr; +} + static zend_always_inline uint32_t zval_refcount_p(const zval* pz) { #if ZEND_DEBUG ZEND_ASSERT(Z_REFCOUNTED_P(pz) || Z_TYPE_P(pz) == IS_ARRAY); From 54f9ca85f398a448b66d316a9e5375c52b530aa0 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Sat, 30 Dec 2017 22:49:26 +0100 Subject: [PATCH 02/10] Move HT iterators count into extra_flags space --- Zend/zend_hash.h | 4 ++-- Zend/zend_types.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Zend/zend_hash.h b/Zend/zend_hash.h index af9c21a7ecc20..249f9bf34077c 100644 --- a/Zend/zend_hash.h +++ b/Zend/zend_hash.h @@ -59,12 +59,12 @@ # define HT_ALLOW_COW_VIOLATION(ht) #endif -#define HT_ITERATORS_COUNT(ht) (ht)->u.v.nIteratorsCount +#define HT_ITERATORS_COUNT(ht) (GC_EXTRA_FLAGS(ht) >> 8) #define HT_ITERATORS_OVERFLOW(ht) (HT_ITERATORS_COUNT(ht) == 0xff) #define HT_HAS_ITERATORS(ht) (HT_ITERATORS_COUNT(ht) != 0) #define HT_SET_ITERATORS_COUNT(ht, iters) \ - do { HT_ITERATORS_COUNT(ht) = (iters); } while (0) + do { GC_EXTRA_FLAGS(ht) = ((iters) << 8) | (GC_EXTRA_FLAGS(ht) & 0xff); } while (0) #define HT_INC_ITERATORS_COUNT(ht) \ HT_SET_ITERATORS_COUNT(ht, HT_ITERATORS_COUNT(ht) + 1) #define HT_DEC_ITERATORS_COUNT(ht) \ diff --git a/Zend/zend_types.h b/Zend/zend_types.h index 9dc3068ee5afc..e2ca3e809f46e 100644 --- a/Zend/zend_types.h +++ b/Zend/zend_types.h @@ -254,8 +254,8 @@ struct _zend_array { ZEND_ENDIAN_LOHI_4( zend_uchar flags, zend_uchar _unused, - zend_uchar nIteratorsCount, - zend_uchar _unused2) + zend_uchar _unused2, + zend_uchar _unused3) } v; uint32_t flags; } u; From cfdccdcfd8dd7088728b62bfdb59b61db3020c2d Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Sat, 30 Dec 2017 23:34:20 +0100 Subject: [PATCH 03/10] Move HT flags into GC extra flags Introduce HT_FLAGS() macro to ease the migration. This brings array size back to what it was. --- Zend/zend_hash.c | 6 +++--- Zend/zend_hash.h | 4 ++-- Zend/zend_types.h | 10 ---------- ext/opcache/zend_persist.c | 3 ++- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/Zend/zend_hash.c b/Zend/zend_hash.c index 0e3155a6d1fa9..264782d6b0bc0 100644 --- a/Zend/zend_hash.c +++ b/Zend/zend_hash.c @@ -159,9 +159,8 @@ static const uint32_t uninitialized_bucket[-HT_MIN_MASK] = ZEND_API const HashTable zend_empty_array = { .gc.rc.refcount = 2, - .gc.rc.u.type_info = IS_ARRAY | (GC_IMMUTABLE << GC_FLAGS_SHIFT), + .gc.rc.u.type_info = IS_ARRAY | (GC_IMMUTABLE << GC_FLAGS_SHIFT) | (HASH_FLAG_STATIC_KEYS << 16), .gc.gc_root = 0, - .u.flags = HASH_FLAG_STATIC_KEYS, .nTableMask = HT_MIN_MASK, .arData = (Bucket*)&uninitialized_bucket[2], .nNumUsed = 0, @@ -1364,7 +1363,8 @@ ZEND_API void ZEND_FASTCALL zend_array_destroy(HashTable *ht) /* break possible cycles */ GC_REMOVE_FROM_BUFFER(ht); - GC_TYPE_INFO(ht) = IS_NULL | (GC_WHITE << GC_FLAGS_SHIFT); + GC_TYPE(ht) = IS_NULL; + GC_FLAGS(ht) = GC_WHITE; if (ht->nNumUsed) { /* In some rare cases destructors of regular arrays may be changed */ diff --git a/Zend/zend_hash.h b/Zend/zend_hash.h index 249f9bf34077c..f3af2d68c3445 100644 --- a/Zend/zend_hash.h +++ b/Zend/zend_hash.h @@ -42,7 +42,7 @@ #define HASH_FLAG_HAS_EMPTY_IND (1<<5) #define HASH_FLAG_ALLOW_COW_VIOLATION (1<<6) -#define HT_FLAGS(ht) (ht)->u.flags +#define HT_FLAGS(ht) GC_EXTRA_FLAGS(ht) #define HT_IS_PACKED(ht) \ ((HT_FLAGS(ht) & HASH_FLAG_PACKED) != 0) @@ -59,7 +59,7 @@ # define HT_ALLOW_COW_VIOLATION(ht) #endif -#define HT_ITERATORS_COUNT(ht) (GC_EXTRA_FLAGS(ht) >> 8) +#define HT_ITERATORS_COUNT(ht) (HT_FLAGS(ht) >> 8) #define HT_ITERATORS_OVERFLOW(ht) (HT_ITERATORS_COUNT(ht) == 0xff) #define HT_HAS_ITERATORS(ht) (HT_ITERATORS_COUNT(ht) != 0) diff --git a/Zend/zend_types.h b/Zend/zend_types.h index e2ca3e809f46e..a9c5029133cfe 100644 --- a/Zend/zend_types.h +++ b/Zend/zend_types.h @@ -249,16 +249,6 @@ typedef struct _zend_array HashTable; struct _zend_array { zend_refcounted_gc_h gc; - union { - struct { - ZEND_ENDIAN_LOHI_4( - zend_uchar flags, - zend_uchar _unused, - zend_uchar _unused2, - zend_uchar _unused3) - } v; - uint32_t flags; - } u; uint32_t nTableMask; Bucket *arData; uint32_t nNumUsed; diff --git a/ext/opcache/zend_persist.c b/ext/opcache/zend_persist.c index f292ce09b0591..d44f453967393 100644 --- a/ext/opcache/zend_persist.c +++ b/ext/opcache/zend_persist.c @@ -367,7 +367,8 @@ static void zend_persist_op_array_ex(zend_op_array *op_array, zend_persistent_sc zend_accel_store(op_array->static_variables, sizeof(HashTable)); /* make immutable array */ GC_SET_REFCOUNT(op_array->static_variables, 2); - GC_TYPE_INFO(op_array->static_variables) = IS_ARRAY | (IS_ARRAY_IMMUTABLE << GC_FLAGS_SHIFT); + GC_TYPE(op_array->static_variables) = IS_ARRAY; + GC_FLAGS(op_array->static_variables) = IS_ARRAY_IMMUTABLE; HT_FLAGS(op_array->static_variables) |= HASH_FLAG_STATIC_KEYS; } } From 4305baf04e4d18cf959fd5aacef879830bfe23ed Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Sun, 31 Dec 2017 13:49:21 +0100 Subject: [PATCH 04/10] Migrate GC to use offsets rather than pointers This is going to make it easier to resize the GC buffer dynamically, as we will not have to take care of migrating all the pointers. It also is a more compact representation on 64-bit systems. On the other hand, the operations become slightly more expensive. This patch preserves the sentinel-based doubly-linked lists. The sentinels now have to be part of the root buffer proper, rather than being separate, so the first two root buffer entries are reserved for the roots and to_free sentinels. This patch leaves two things in a broken state: * additional_buffer doesn't work. Managing references into separate buffer allocations seems too complicated and inefficient to me, now that we have the ability to reallocate the buffer. I plan to fix this by increasing the buffer size if we need the space to store additional entries during GC. * recursive GC may not work correctly, as the to_free list is no longer stored locally during the actual GC procedure. I plan to fix this by not allowing recursive GC and instead just collecting garbage, as a dynamically resized root buffer can guarantee that we don't lose anything. --- Zend/zend_gc.c | 268 +++++++++++++++++++++++++++---------------------- Zend/zend_gc.h | 26 +++-- 2 files changed, 165 insertions(+), 129 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index a32acb27e1a4d..89f3853d7b6ab 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -104,6 +104,20 @@ ZEND_API int (*gc_collect_cycles)(void); # define GC_TRACE(str) #endif +#define GC_INVALID ((uint32_t) -1) +#define GC_ROOTS_SENTINEL ((uint32_t) 0) +#define GC_TO_FREE_SENTINEL ((uint32_t) 1) +#define GC_FIRST_REAL_ROOT ((uint32_t) 2) + +#define GC_TO_BUF(addr) (GC_G(buf) + (addr)) +#define GC_TO_ADDR(buffer) ((buffer) - GC_G(buf)) + +#define GC_ROOTS() GC_TO_BUF(GC_ROOTS_SENTINEL) +#define GC_TO_FREE() GC_TO_BUF(GC_TO_FREE_SENTINEL) + +#define GC_NEXT_BUF(buf) GC_TO_BUF((buf)->next) +#define GC_PREV_BUF(buf) GC_TO_BUF((buf)->prev) + #define GC_REF_GET_COLOR(ref) \ (GC_FLAGS(ref) & GC_COLOR) #define GC_REF_SET_COLOR_EX(ref, c) \ @@ -144,34 +158,41 @@ static void gc_trace_ref(zend_refcounted *ref) { gc_color_name(GC_REF_GET_COLOR(ref)), zend_hash_num_elements(arr)); } else { - fprintf(stderr, "[%p] rc=%d addr=%d %s %s ", - ref, GC_REFCOUNT(ref), GC_ADDRESS(ref), + fprintf(stderr, "[%p] rc=%d %s %s ", + ref, GC_REFCOUNT(ref), gc_color_name(GC_REF_GET_COLOR(ref)), zend_get_type_by_const(GC_TYPE(ref))); } } #endif +static const gc_root_buffer gc_dummy_buffer[2] = { + { NULL, GC_ROOTS_SENTINEL, GC_ROOTS_SENTINEL, 0 }, + { NULL, GC_TO_FREE_SENTINEL, GC_TO_FREE_SENTINEL, 0 }, +}; +#define GC_DUMMY_BUF() ((gc_root_buffer *) &gc_dummy_buffer) + static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root) { - root->next->prev = root->prev; - root->prev->next = root->next; + GC_NEXT_BUF(root)->prev = root->prev; + GC_PREV_BUF(root)->next = root->next; root->prev = GC_G(unused); - GC_G(unused) = root; + GC_G(unused) = GC_TO_ADDR(root); GC_BENCH_DEC(root_buf_length); } static zend_always_inline void gc_remove_from_additional_roots(gc_root_buffer *root) { - root->next->prev = root->prev; - root->prev->next = root->next; + /* TODO */ + /*root->next->prev = root->prev; + root->prev->next = root->next;*/ } static void root_buffer_dtor(zend_gc_globals *gc_globals) { - if (gc_globals->buf) { + if (gc_globals->buf != GC_DUMMY_BUF()) { free(gc_globals->buf); - gc_globals->buf = NULL; + gc_globals->buf = GC_DUMMY_BUF(); } } @@ -180,15 +201,12 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) gc_globals->gc_enabled = 0; gc_globals->gc_active = 0; - gc_globals->buf = NULL; + gc_globals->buf = GC_DUMMY_BUF(); - gc_globals->roots.next = &gc_globals->roots; - gc_globals->roots.prev = &gc_globals->roots; - gc_globals->unused = NULL; - gc_globals->next_to_free = NULL; - - gc_globals->to_free.next = &gc_globals->to_free; - gc_globals->to_free.prev = &gc_globals->to_free; + gc_globals->unused = GC_INVALID; + gc_globals->first_unused = GC_FIRST_REAL_ROOT; + gc_globals->last_unused = GC_FIRST_REAL_ROOT; + gc_globals->next_to_free = GC_INVALID; gc_globals->gc_runs = 0; gc_globals->collected = 0; @@ -236,35 +254,36 @@ ZEND_API void gc_reset(void) GC_G(zval_marked_grey) = 0; #endif - GC_G(roots).next = &GC_G(roots); - GC_G(roots).prev = &GC_G(roots); - - GC_G(to_free).next = &GC_G(to_free); - GC_G(to_free).prev = &GC_G(to_free); - - if (GC_G(buf)) { - GC_G(unused) = NULL; - GC_G(first_unused) = GC_G(buf) + 1; - } else { - GC_G(unused) = NULL; - GC_G(first_unused) = NULL; - GC_G(last_unused) = NULL; + if (GC_G(buf) != GC_DUMMY_BUF()) { + GC_ROOTS()->next = GC_ROOTS_SENTINEL; + GC_ROOTS()->prev = GC_ROOTS_SENTINEL; + GC_TO_FREE()->next = GC_TO_FREE_SENTINEL; + GC_TO_FREE()->prev = GC_TO_FREE_SENTINEL; } + GC_G(unused) = GC_INVALID; + GC_G(first_unused) = GC_FIRST_REAL_ROOT; + GC_G(additional_buffer) = NULL; } ZEND_API void gc_init(void) { - if (GC_G(buf) == NULL && GC_G(gc_enabled)) { - GC_G(buf) = (gc_root_buffer*) malloc(sizeof(gc_root_buffer) * GC_ROOT_BUFFER_MAX_ENTRIES); - GC_G(last_unused) = &GC_G(buf)[GC_ROOT_BUFFER_MAX_ENTRIES]; + if (GC_G(buf) == GC_DUMMY_BUF() && GC_G(gc_enabled)) { + GC_G(buf) = malloc(sizeof(gc_root_buffer) * GC_ROOT_BUFFER_MAX_ENTRIES); + if (GC_G(buf)) { + GC_G(last_unused) = GC_ROOT_BUFFER_MAX_ENTRIES; + } else { + GC_G(buf) = GC_DUMMY_BUF(); + GC_G(last_unused) = GC_FIRST_REAL_ROOT; + } gc_reset(); } } ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) { + uint32_t newRootAddr; gc_root_buffer *newRoot; if (UNEXPECTED(CG(unclean_shutdown)) || UNEXPECTED(GC_G(gc_active))) { @@ -277,11 +296,13 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) GC_BENCH_INC(zval_possible_root); - newRoot = GC_G(unused); - if (newRoot) { + newRootAddr = GC_G(unused); + if (newRootAddr != GC_INVALID) { + newRoot = GC_TO_BUF(newRootAddr); GC_G(unused) = newRoot->prev; } else if (GC_G(first_unused) != GC_G(last_unused)) { - newRoot = GC_G(first_unused); + newRootAddr = GC_G(first_unused); + newRoot = GC_TO_BUF(newRootAddr); GC_G(first_unused)++; } else { if (!GC_G(gc_enabled)) { @@ -297,8 +318,9 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) if (UNEXPECTED(GC_ADDRESS(ref))) { return; } - newRoot = GC_G(unused); - if (!newRoot) { + + newRootAddr = GC_G(unused); + if (newRootAddr == GC_INVALID) { #if ZEND_GC_DEBUG if (!GC_G(gc_full)) { fprintf(stderr, "GC: no space to record new root candidate\n"); @@ -307,18 +329,22 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) #endif return; } + newRoot = GC_TO_BUF(newRootAddr); GC_G(unused) = newRoot->prev; } GC_TRACE_SET_COLOR(ref, GC_PURPLE); - GC_SET_ADDRESS(ref, newRoot - GC_G(buf)); + GC_SET_ADDRESS(ref, newRootAddr); GC_REF_SET_PURPLE(ref); newRoot->ref = ref; - newRoot->next = GC_G(roots).next; - newRoot->prev = &GC_G(roots); - GC_G(roots).next->prev = newRoot; - GC_G(roots).next = newRoot; + { + gc_root_buffer *roots = GC_ROOTS(); + newRoot->next = roots->next; + newRoot->prev = GC_ROOTS_SENTINEL; + GC_NEXT_BUF(roots)->prev = newRootAddr; + roots->next = newRootAddr; + } GC_BENCH_INC(zval_buffered); GC_BENCH_INC(root_buf_length); @@ -347,14 +373,16 @@ static zend_always_inline gc_root_buffer* gc_find_additional_buffer(zend_refcoun ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) { + uint32_t addr; gc_root_buffer *root; ZEND_ASSERT(GC_ADDRESS(ref)); GC_BENCH_INC(zval_remove_from_buffer); - if (EXPECTED(GC_ADDRESS(ref) < GC_ROOT_BUFFER_MAX_ENTRIES)) { - root = GC_G(buf) + GC_ADDRESS(ref); + addr = GC_ADDRESS(ref); + if (EXPECTED(addr < GC_ROOT_BUFFER_MAX_ENTRIES)) { + root = GC_TO_BUF(addr); gc_remove_from_roots(root); } else { root = gc_find_additional_buffer(ref); @@ -366,7 +394,7 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) GC_SET_ADDRESS(ref, 0); /* updete next root that is going to be freed */ - if (GC_G(next_to_free) == root) { + if (GC_G(next_to_free) == addr) { GC_G(next_to_free) = root->next; } } @@ -582,13 +610,14 @@ static void gc_mark_grey(zend_refcounted *ref) static void gc_mark_roots(void) { - gc_root_buffer *current = GC_G(roots).next; + gc_root_buffer *roots = GC_ROOTS(); + gc_root_buffer *current = GC_NEXT_BUF(roots); - while (current != &GC_G(roots)) { + while (current != roots) { if (GC_REF_GET_COLOR(current->ref) == GC_PURPLE) { gc_mark_grey(current->ref); } - current = current->next; + current = GC_NEXT_BUF(current); } } @@ -691,27 +720,29 @@ static void gc_scan(zend_refcounted *ref) static void gc_scan_roots(void) { - gc_root_buffer *current = GC_G(roots).next; + gc_root_buffer *roots = GC_ROOTS(); + gc_root_buffer *current = GC_NEXT_BUF(roots); - while (current != &GC_G(roots)) { + while (current != roots) { gc_scan(current->ref); - current = current->next; + current = GC_NEXT_BUF(current); } } static void gc_add_garbage(zend_refcounted *ref) { - gc_root_buffer *buf = GC_G(unused); + gc_root_buffer *roots = GC_ROOTS(); + uint32_t addr; + gc_root_buffer *buf; - if (buf) { + if (GC_G(unused) != GC_INVALID) { + addr = GC_G(unused); + buf = GC_TO_BUF(addr); GC_G(unused) = buf->prev; - /* optimization: color is already GC_BLACK (0) */ - GC_SET_ADDRESS(ref, buf - GC_G(buf)); } else if (GC_G(first_unused) != GC_G(last_unused)) { - buf = GC_G(first_unused); + addr = GC_G(first_unused); + buf = GC_TO_BUF(addr); GC_G(first_unused)++; - /* optimization: color is already GC_BLACK (0) */ - GC_SET_ADDRESS(ref, buf - GC_G(buf)); } else { /* If we don't have free slots in the buffer, allocate a new one and * set it's address above GC_ROOT_BUFFER_MAX_ENTRIES that have special @@ -723,18 +754,20 @@ static void gc_add_garbage(zend_refcounted *ref) new_buffer->next = GC_G(additional_buffer); GC_G(additional_buffer) = new_buffer; } + addr = GC_ROOT_BUFFER_MAX_ENTRIES + GC_G(additional_buffer)->used; buf = GC_G(additional_buffer)->buf + GC_G(additional_buffer)->used; - /* optimization: color is already GC_BLACK (0) */ - GC_SET_ADDRESS(ref, GC_ROOT_BUFFER_MAX_ENTRIES + GC_G(additional_buffer)->used); GC_G(additional_buffer)->used++; + /* TODO Not going to work */ } - if (buf) { - buf->ref = ref; - buf->next = GC_G(roots).next; - buf->prev = &GC_G(roots); - GC_G(roots).next->prev = buf; - GC_G(roots).next = buf; - } + + /* optimization: color is already GC_BLACK (0) */ + GC_SET_ADDRESS(ref, addr); + + buf->ref = ref; + buf->next = roots->next; + buf->prev = GC_ROOTS_SENTINEL; + GC_NEXT_BUF(roots)->prev = addr; + roots->next = addr; } static int gc_collect_white(zend_refcounted *ref, uint32_t *flags) @@ -867,11 +900,12 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags) static int gc_collect_roots(uint32_t *flags) { int count = 0; - gc_root_buffer *current = GC_G(roots).next; + gc_root_buffer *roots = GC_ROOTS(); + gc_root_buffer *current = GC_NEXT_BUF(roots); /* remove non-garbage from the list */ - while (current != &GC_G(roots)) { - gc_root_buffer *next = current->next; + while (current != roots) { + gc_root_buffer *next = GC_NEXT_BUF(current); if (GC_REF_GET_COLOR(current->ref) == GC_BLACK) { if (EXPECTED(GC_ADDRESS(current->ref) < GC_ROOT_BUFFER_MAX_ENTRIES)) { gc_remove_from_roots(current); @@ -883,32 +917,33 @@ static int gc_collect_roots(uint32_t *flags) current = next; } - current = GC_G(roots).next; - while (current != &GC_G(roots)) { + current = GC_NEXT_BUF(roots); + while (current != roots) { if (GC_REF_GET_COLOR(current->ref) == GC_WHITE) { count += gc_collect_white(current->ref, flags); } - current = current->next; + current = GC_NEXT_BUF(current); } /* relink remaining roots into list to free */ - if (GC_G(roots).next != &GC_G(roots)) { - if (GC_G(to_free).next == &GC_G(to_free)) { + if (roots->next != GC_ROOTS_SENTINEL) { + gc_root_buffer *to_free = GC_TO_FREE(); + if (to_free->next == GC_TO_FREE_SENTINEL) { /* move roots into list to free */ - GC_G(to_free).next = GC_G(roots).next; - GC_G(to_free).prev = GC_G(roots).prev; - GC_G(to_free).next->prev = &GC_G(to_free); - GC_G(to_free).prev->next = &GC_G(to_free); + to_free->next = roots->next; + to_free->prev = roots->prev; + GC_NEXT_BUF(to_free)->prev = GC_TO_FREE_SENTINEL; + GC_PREV_BUF(to_free)->next = GC_TO_FREE_SENTINEL; } else { /* add roots into list to free */ - GC_G(to_free).prev->next = GC_G(roots).next; - GC_G(roots).next->prev = GC_G(to_free).prev; - GC_G(roots).prev->next = &GC_G(to_free); - GC_G(to_free).prev = GC_G(roots).prev; + GC_PREV_BUF(to_free)->next = roots->next; + GC_NEXT_BUF(roots)->prev = to_free->prev; + GC_PREV_BUF(roots)->next = GC_TO_FREE_SENTINEL; + to_free->prev = roots->prev; } - GC_G(roots).next = &GC_G(roots); - GC_G(roots).prev = &GC_G(roots); + roots->next = GC_ROOTS_SENTINEL; + roots->prev = GC_ROOTS_SENTINEL; } return count; } @@ -1019,11 +1054,12 @@ static void gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buff ZEND_API int zend_gc_collect_cycles(void) { int count = 0; + gc_root_buffer *roots = GC_ROOTS(); - if (GC_G(roots).next != &GC_G(roots)) { - gc_root_buffer *current, *next, *orig_next_to_free; + if (roots->next != GC_ROOTS_SENTINEL) { + gc_root_buffer *current, *next; zend_refcounted *p; - gc_root_buffer to_free; + gc_root_buffer *to_free; uint32_t gc_flags = 0; gc_additional_buffer *additional_buffer_snapshot; #if ZEND_GC_DEBUG @@ -1054,26 +1090,17 @@ ZEND_API int zend_gc_collect_cycles(void) #if ZEND_GC_DEBUG GC_G(gc_full) = orig_gc_full; #endif + + /* TODO: Handle nested GC */ GC_G(gc_active) = 0; - if (GC_G(to_free).next == &GC_G(to_free)) { + to_free = GC_TO_FREE(); + if (to_free->next == GC_TO_FREE_SENTINEL) { /* nothing to free */ GC_TRACE("Nothing to free"); return 0; } - /* Copy global to_free list into local list */ - to_free.next = GC_G(to_free).next; - to_free.prev = GC_G(to_free).prev; - to_free.next->prev = &to_free; - to_free.prev->next = &to_free; - - /* Free global list */ - GC_G(to_free).next = &GC_G(to_free); - GC_G(to_free).prev = &GC_G(to_free); - - orig_next_to_free = GC_G(next_to_free); - #if ZEND_GC_DEBUG orig_gc_full = GC_G(gc_full); GC_G(gc_full) = 0; @@ -1083,15 +1110,15 @@ ZEND_API int zend_gc_collect_cycles(void) GC_TRACE("Calling destructors"); /* Remember reference counters before calling destructors */ - current = to_free.next; - while (current != &to_free) { + current = GC_NEXT_BUF(to_free); + while (current != to_free) { current->refcount = GC_REFCOUNT(current->ref); - current = current->next; + current = GC_NEXT_BUF(current); } /* Call destructors */ - current = to_free.next; - while (current != &to_free) { + current = GC_NEXT_BUF(to_free); + while (current != to_free) { p = current->ref; GC_G(next_to_free) = current->next; if (GC_TYPE(p) == IS_OBJECT) { @@ -1109,25 +1136,25 @@ ZEND_API int zend_gc_collect_cycles(void) } } } - current = GC_G(next_to_free); + current = GC_NEXT_BUF(current); } /* Remove values captured in destructors */ - current = to_free.next; - while (current != &to_free) { + current = GC_NEXT_BUF(to_free); + while (current != to_free) { GC_G(next_to_free) = current->next; if (GC_REFCOUNT(current->ref) > current->refcount) { gc_remove_nested_data_from_buffer(current->ref, current); } - current = GC_G(next_to_free); + current = GC_NEXT_BUF(current); } } /* Destroy zvals */ GC_TRACE("Destroying zvals"); GC_G(gc_active) = 1; - current = to_free.next; - while (current != &to_free) { + current = GC_NEXT_BUF(to_free); + while (current != to_free) { p = current->ref; GC_G(next_to_free) = current->next; GC_TRACE_REF(p, "destroying"); @@ -1157,22 +1184,25 @@ ZEND_API int zend_gc_collect_cycles(void) zend_hash_destroy(arr); } - current = GC_G(next_to_free); + current = GC_TO_BUF(GC_G(next_to_free)); } /* Free objects */ - current = to_free.next; - while (current != &to_free) { - next = current->next; + current = GC_NEXT_BUF(to_free); + while (current != to_free) { + next = GC_NEXT_BUF(current); p = current->ref; if (EXPECTED(current >= GC_G(buf) && current < GC_G(buf) + GC_ROOT_BUFFER_MAX_ENTRIES)) { current->prev = GC_G(unused); - GC_G(unused) = current; + GC_G(unused) = GC_TO_ADDR(current); } efree(p); current = next; } + to_free->next = GC_TO_FREE_SENTINEL; + to_free->prev = GC_TO_FREE_SENTINEL; + while (GC_G(additional_buffer) != additional_buffer_snapshot) { gc_additional_buffer *next = GC_G(additional_buffer)->next; efree(GC_G(additional_buffer)); @@ -1181,7 +1211,7 @@ ZEND_API int zend_gc_collect_cycles(void) GC_TRACE("Collection finished"); GC_G(collected) += count; - GC_G(next_to_free) = orig_next_to_free; + GC_G(next_to_free) = GC_INVALID; #if ZEND_GC_DEBUG GC_G(gc_full) = orig_gc_full; #endif diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index bec464124dfc0..935994adf2538 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -41,12 +41,18 @@ #endif typedef struct _gc_root_buffer { - zend_refcounted *ref; - struct _gc_root_buffer *next; /* double-linked list */ - struct _gc_root_buffer *prev; - uint32_t refcount; + zend_refcounted *ref; + uint32_t next; /* double-linked list */ + uint32_t prev; + uint32_t refcount; } gc_root_buffer; +/* Doubly linked list of root buffers, lists are terminated with GC_INVALID */ +typedef struct _gc_root_list { + uint32_t next; + uint32_t prev; +} gc_root_list; + #define GC_NUM_ADDITIONAL_ENTRIES \ ((4096 - ZEND_MM_OVERHEAD - sizeof(void*) * 2) / sizeof(gc_root_buffer)) @@ -64,13 +70,13 @@ typedef struct _zend_gc_globals { zend_bool gc_full; gc_root_buffer *buf; /* preallocated arrays of buffers */ - gc_root_buffer roots; /* list of possible roots of cycles */ - gc_root_buffer *unused; /* list of unused buffers */ - gc_root_buffer *first_unused; /* pointer to first unused buffer */ - gc_root_buffer *last_unused; /* pointer to last unused buffer */ + //gc_root_list roots; /* list of possible roots of cycles */ + uint32_t unused; /* linked list of unused buffers */ + uint32_t first_unused; /* first unused buffer */ + uint32_t last_unused; /* last unused buffer */ - gc_root_buffer to_free; /* list to free */ - gc_root_buffer *next_to_free; + //gc_root_list to_free; /* list to free */ + uint32_t next_to_free; /* next to free in to_free list */ uint32_t gc_runs; uint32_t collected; From b61229b681963b0fc2dfd291a275ab68de56c08a Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 18 Jan 2018 23:18:36 +0100 Subject: [PATCH 05/10] Remove additional GC buffers in favor of resizing If we don't have enough space to store additional garbage, resize the root buffer. The resizing heuristic is just a dummy for now. --- Zend/zend_gc.c | 118 ++++++++++++++++--------------------------------- Zend/zend_gc.h | 23 ---------- 2 files changed, 37 insertions(+), 104 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index 89f3853d7b6ab..278c033e0c09a 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -72,8 +72,8 @@ #include "zend.h" #include "zend_API.h" -/* one (0) is reserved */ -#define GC_ROOT_BUFFER_MAX_ENTRIES 10001 +#define GC_ROOT_BUFFER_DEFAULT_SIZE 10002 +#define GC_ROOT_BUFFER_SIZE_INCREMENT 4000 // TODO Arbitrary number #define GC_HAS_DESTRUCTORS (1<<0) @@ -181,13 +181,6 @@ static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root) GC_BENCH_DEC(root_buf_length); } -static zend_always_inline void gc_remove_from_additional_roots(gc_root_buffer *root) -{ - /* TODO */ - /*root->next->prev = root->prev; - root->prev->next = root->next;*/ -} - static void root_buffer_dtor(zend_gc_globals *gc_globals) { if (gc_globals->buf != GC_DUMMY_BUF()) { @@ -211,8 +204,6 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) gc_globals->gc_runs = 0; gc_globals->collected = 0; - gc_globals->additional_buffer = NULL; - #if GC_BENCH gc_globals->root_buf_length = 0; gc_globals->root_buf_peak = 0; @@ -263,16 +254,14 @@ ZEND_API void gc_reset(void) GC_G(unused) = GC_INVALID; GC_G(first_unused) = GC_FIRST_REAL_ROOT; - - GC_G(additional_buffer) = NULL; } ZEND_API void gc_init(void) { if (GC_G(buf) == GC_DUMMY_BUF() && GC_G(gc_enabled)) { - GC_G(buf) = malloc(sizeof(gc_root_buffer) * GC_ROOT_BUFFER_MAX_ENTRIES); + GC_G(buf) = malloc(sizeof(gc_root_buffer) * GC_ROOT_BUFFER_DEFAULT_SIZE); if (GC_G(buf)) { - GC_G(last_unused) = GC_ROOT_BUFFER_MAX_ENTRIES; + GC_G(last_unused) = GC_ROOT_BUFFER_DEFAULT_SIZE; } else { GC_G(buf) = GC_DUMMY_BUF(); GC_G(last_unused) = GC_FIRST_REAL_ROOT; @@ -281,6 +270,18 @@ ZEND_API void gc_init(void) } } +static void gc_resize_root_buffer(size_t new_size) { + /* Only allow increasing for now */ + ZEND_ASSERT(new_size > GC_G(last_unused)); + GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1); + GC_G(buf) = realloc(GC_G(buf), sizeof(gc_root_buffer) * new_size); + GC_G(last_unused) = new_size; +} + +static void gc_grow_root_buffer() { + gc_resize_root_buffer(GC_G(last_unused) + GC_ROOT_BUFFER_SIZE_INCREMENT); +} + ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) { uint32_t newRootAddr; @@ -306,6 +307,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) GC_G(first_unused)++; } else { if (!GC_G(gc_enabled)) { + /* TODO Resize here */ return; } GC_ADDREF(ref); @@ -321,6 +323,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) newRootAddr = GC_G(unused); if (newRootAddr == GC_INVALID) { + /* TODO Resize here */ #if ZEND_GC_DEBUG if (!GC_G(gc_full)) { fprintf(stderr, "GC: no space to record new root candidate\n"); @@ -351,26 +354,6 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) GC_BENCH_PEAK(root_buf_peak, root_buf_length); } -static zend_always_inline gc_root_buffer* gc_find_additional_buffer(zend_refcounted *ref) -{ - gc_additional_buffer *additional_buffer = GC_G(additional_buffer); - - /* We have to check each additional_buffer to find which one holds the ref */ - while (additional_buffer) { - uint32_t idx = GC_ADDRESS(ref) - GC_ROOT_BUFFER_MAX_ENTRIES; - if (idx < additional_buffer->used) { - gc_root_buffer *root = additional_buffer->buf + idx; - if (root->ref == ref) { - return root; - } - } - additional_buffer = additional_buffer->next; - } - - ZEND_ASSERT(0); - return NULL; -} - ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) { uint32_t addr; @@ -381,13 +364,9 @@ ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) GC_BENCH_INC(zval_remove_from_buffer); addr = GC_ADDRESS(ref); - if (EXPECTED(addr < GC_ROOT_BUFFER_MAX_ENTRIES)) { - root = GC_TO_BUF(addr); - gc_remove_from_roots(root); - } else { - root = gc_find_additional_buffer(ref); - gc_remove_from_additional_roots(root); - } + root = GC_TO_BUF(addr); + gc_remove_from_roots(root); + if (GC_REF_GET_COLOR(ref) != GC_BLACK) { GC_TRACE_SET_COLOR(ref, GC_PURPLE); } @@ -731,7 +710,7 @@ static void gc_scan_roots(void) static void gc_add_garbage(zend_refcounted *ref) { - gc_root_buffer *roots = GC_ROOTS(); + gc_root_buffer *roots; uint32_t addr; gc_root_buffer *buf; @@ -744,25 +723,16 @@ static void gc_add_garbage(zend_refcounted *ref) buf = GC_TO_BUF(addr); GC_G(first_unused)++; } else { - /* If we don't have free slots in the buffer, allocate a new one and - * set it's address above GC_ROOT_BUFFER_MAX_ENTRIES that have special - * meaning. - */ - if (!GC_G(additional_buffer) || GC_G(additional_buffer)->used == GC_NUM_ADDITIONAL_ENTRIES) { - gc_additional_buffer *new_buffer = emalloc(sizeof(gc_additional_buffer)); - new_buffer->used = 0; - new_buffer->next = GC_G(additional_buffer); - GC_G(additional_buffer) = new_buffer; - } - addr = GC_ROOT_BUFFER_MAX_ENTRIES + GC_G(additional_buffer)->used; - buf = GC_G(additional_buffer)->buf + GC_G(additional_buffer)->used; - GC_G(additional_buffer)->used++; - /* TODO Not going to work */ + gc_grow_root_buffer(); + addr = GC_G(first_unused); + buf = GC_TO_BUF(addr); + GC_G(first_unused)++; } /* optimization: color is already GC_BLACK (0) */ GC_SET_ADDRESS(ref, addr); + roots = GC_ROOTS(); buf->ref = ref; buf->next = roots->next; buf->prev = GC_ROOTS_SENTINEL; @@ -907,25 +877,25 @@ static int gc_collect_roots(uint32_t *flags) while (current != roots) { gc_root_buffer *next = GC_NEXT_BUF(current); if (GC_REF_GET_COLOR(current->ref) == GC_BLACK) { - if (EXPECTED(GC_ADDRESS(current->ref) < GC_ROOT_BUFFER_MAX_ENTRIES)) { - gc_remove_from_roots(current); - } else { - gc_remove_from_additional_roots(current); - } + gc_remove_from_roots(current); GC_SET_ADDRESS(current->ref, 0); /* reset GC_ADDRESS() and keep GC_BLACK */ } current = next; } + /* Root buffer might be reallocated during gc_collect_white, + * make sure to reload pointers. */ current = GC_NEXT_BUF(roots); - while (current != roots) { + while (current != GC_ROOTS()) { + uint32_t next = current->next; if (GC_REF_GET_COLOR(current->ref) == GC_WHITE) { count += gc_collect_white(current->ref, flags); } - current = GC_NEXT_BUF(current); + current = GC_TO_BUF(next); } /* relink remaining roots into list to free */ + roots = GC_ROOTS(); if (roots->next != GC_ROOTS_SENTINEL) { gc_root_buffer *to_free = GC_TO_FREE(); if (to_free->next == GC_TO_FREE_SENTINEL) { @@ -961,11 +931,7 @@ static void gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buff GC_REF_GET_COLOR(ref) == GC_BLACK)) { GC_TRACE_REF(ref, "removing from buffer"); if (root) { - if (EXPECTED(GC_ADDRESS(root->ref) < GC_ROOT_BUFFER_MAX_ENTRIES)) { - gc_remove_from_roots(root); - } else { - gc_remove_from_additional_roots(root); - } + gc_remove_from_roots(root); GC_SET_ADDRESS(ref, 0); root = NULL; } else { @@ -1061,7 +1027,6 @@ ZEND_API int zend_gc_collect_cycles(void) zend_refcounted *p; gc_root_buffer *to_free; uint32_t gc_flags = 0; - gc_additional_buffer *additional_buffer_snapshot; #if ZEND_GC_DEBUG zend_bool orig_gc_full; #endif @@ -1085,7 +1050,6 @@ ZEND_API int zend_gc_collect_cycles(void) #endif GC_TRACE("Collecting roots"); - additional_buffer_snapshot = GC_G(additional_buffer); count = gc_collect_roots(&gc_flags); #if ZEND_GC_DEBUG GC_G(gc_full) = orig_gc_full; @@ -1192,10 +1156,8 @@ ZEND_API int zend_gc_collect_cycles(void) while (current != to_free) { next = GC_NEXT_BUF(current); p = current->ref; - if (EXPECTED(current >= GC_G(buf) && current < GC_G(buf) + GC_ROOT_BUFFER_MAX_ENTRIES)) { - current->prev = GC_G(unused); - GC_G(unused) = GC_TO_ADDR(current); - } + current->prev = GC_G(unused); + GC_G(unused) = GC_TO_ADDR(current); efree(p); current = next; } @@ -1203,12 +1165,6 @@ ZEND_API int zend_gc_collect_cycles(void) to_free->next = GC_TO_FREE_SENTINEL; to_free->prev = GC_TO_FREE_SENTINEL; - while (GC_G(additional_buffer) != additional_buffer_snapshot) { - gc_additional_buffer *next = GC_G(additional_buffer)->next; - efree(GC_G(additional_buffer)); - GC_G(additional_buffer) = next; - } - GC_TRACE("Collection finished"); GC_G(collected) += count; GC_G(next_to_free) = GC_INVALID; diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index 935994adf2538..972b81682255d 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -47,35 +47,15 @@ typedef struct _gc_root_buffer { uint32_t refcount; } gc_root_buffer; -/* Doubly linked list of root buffers, lists are terminated with GC_INVALID */ -typedef struct _gc_root_list { - uint32_t next; - uint32_t prev; -} gc_root_list; - -#define GC_NUM_ADDITIONAL_ENTRIES \ - ((4096 - ZEND_MM_OVERHEAD - sizeof(void*) * 2) / sizeof(gc_root_buffer)) - -typedef struct _gc_additional_bufer gc_additional_buffer; - -struct _gc_additional_bufer { - uint32_t used; - gc_additional_buffer *next; - gc_root_buffer buf[GC_NUM_ADDITIONAL_ENTRIES]; -}; - typedef struct _zend_gc_globals { zend_bool gc_enabled; zend_bool gc_active; zend_bool gc_full; gc_root_buffer *buf; /* preallocated arrays of buffers */ - //gc_root_list roots; /* list of possible roots of cycles */ uint32_t unused; /* linked list of unused buffers */ uint32_t first_unused; /* first unused buffer */ uint32_t last_unused; /* last unused buffer */ - - //gc_root_list to_free; /* list to free */ uint32_t next_to_free; /* next to free in to_free list */ uint32_t gc_runs; @@ -89,9 +69,6 @@ typedef struct _zend_gc_globals { uint32_t zval_remove_from_buffer; uint32_t zval_marked_grey; #endif - - gc_additional_buffer *additional_buffer; - } zend_gc_globals; #ifdef ZTS From 3874eaa1eee67d475d83217974a94b8053848f28 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 18 Jan 2018 23:33:36 +0100 Subject: [PATCH 06/10] Rename last_unused to buf_size --- Zend/zend_gc.c | 19 ++++++++++--------- Zend/zend_gc.h | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index 278c033e0c09a..bf1663233d70d 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -108,6 +108,7 @@ ZEND_API int (*gc_collect_cycles)(void); #define GC_ROOTS_SENTINEL ((uint32_t) 0) #define GC_TO_FREE_SENTINEL ((uint32_t) 1) #define GC_FIRST_REAL_ROOT ((uint32_t) 2) +#define GC_DUMMY_BUF_SIZE GC_FIRST_REAL_ROOT #define GC_TO_BUF(addr) (GC_G(buf) + (addr)) #define GC_TO_ADDR(buffer) ((buffer) - GC_G(buf)) @@ -166,7 +167,7 @@ static void gc_trace_ref(zend_refcounted *ref) { } #endif -static const gc_root_buffer gc_dummy_buffer[2] = { +static const gc_root_buffer gc_dummy_buffer[GC_DUMMY_BUF_SIZE] = { { NULL, GC_ROOTS_SENTINEL, GC_ROOTS_SENTINEL, 0 }, { NULL, GC_TO_FREE_SENTINEL, GC_TO_FREE_SENTINEL, 0 }, }; @@ -198,7 +199,7 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) gc_globals->unused = GC_INVALID; gc_globals->first_unused = GC_FIRST_REAL_ROOT; - gc_globals->last_unused = GC_FIRST_REAL_ROOT; + gc_globals->buf_size = GC_DUMMY_BUF_SIZE; gc_globals->next_to_free = GC_INVALID; gc_globals->gc_runs = 0; @@ -261,10 +262,10 @@ ZEND_API void gc_init(void) if (GC_G(buf) == GC_DUMMY_BUF() && GC_G(gc_enabled)) { GC_G(buf) = malloc(sizeof(gc_root_buffer) * GC_ROOT_BUFFER_DEFAULT_SIZE); if (GC_G(buf)) { - GC_G(last_unused) = GC_ROOT_BUFFER_DEFAULT_SIZE; + GC_G(buf_size) = GC_ROOT_BUFFER_DEFAULT_SIZE; } else { GC_G(buf) = GC_DUMMY_BUF(); - GC_G(last_unused) = GC_FIRST_REAL_ROOT; + GC_G(buf_size) = GC_DUMMY_BUF_SIZE; } gc_reset(); } @@ -272,14 +273,14 @@ ZEND_API void gc_init(void) static void gc_resize_root_buffer(size_t new_size) { /* Only allow increasing for now */ - ZEND_ASSERT(new_size > GC_G(last_unused)); + ZEND_ASSERT(new_size > GC_G(buf_size)); GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1); GC_G(buf) = realloc(GC_G(buf), sizeof(gc_root_buffer) * new_size); - GC_G(last_unused) = new_size; + GC_G(buf_size) = new_size; } static void gc_grow_root_buffer() { - gc_resize_root_buffer(GC_G(last_unused) + GC_ROOT_BUFFER_SIZE_INCREMENT); + gc_resize_root_buffer(GC_G(buf_size) + GC_ROOT_BUFFER_SIZE_INCREMENT); } ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) @@ -301,7 +302,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) if (newRootAddr != GC_INVALID) { newRoot = GC_TO_BUF(newRootAddr); GC_G(unused) = newRoot->prev; - } else if (GC_G(first_unused) != GC_G(last_unused)) { + } else if (GC_G(first_unused) != GC_G(buf_size)) { newRootAddr = GC_G(first_unused); newRoot = GC_TO_BUF(newRootAddr); GC_G(first_unused)++; @@ -718,7 +719,7 @@ static void gc_add_garbage(zend_refcounted *ref) addr = GC_G(unused); buf = GC_TO_BUF(addr); GC_G(unused) = buf->prev; - } else if (GC_G(first_unused) != GC_G(last_unused)) { + } else if (GC_G(first_unused) != GC_G(buf_size)) { addr = GC_G(first_unused); buf = GC_TO_BUF(addr); GC_G(first_unused)++; diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index 972b81682255d..3acf42e9d8f49 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -53,9 +53,9 @@ typedef struct _zend_gc_globals { zend_bool gc_full; gc_root_buffer *buf; /* preallocated arrays of buffers */ + uint32_t buf_size; /* size of the GC buffer */ uint32_t unused; /* linked list of unused buffers */ uint32_t first_unused; /* first unused buffer */ - uint32_t last_unused; /* last unused buffer */ uint32_t next_to_free; /* next to free in to_free list */ uint32_t gc_runs; From 1eda5fa45799dcc2bf198203be458ba0a29b71d8 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Thu, 18 Jan 2018 23:54:27 +0100 Subject: [PATCH 07/10] Make collection threshold and buffer size independent Keep track of the number of roots and collect based on that, instead of going by the buffer size. --- Zend/zend_gc.c | 27 ++++++++++++++++++++++++++- Zend/zend_gc.h | 1 + 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index bf1663233d70d..6895ece142113 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -72,6 +72,7 @@ #include "zend.h" #include "zend_API.h" +#define GC_COLLECTION_THRESHOLD 10000 #define GC_ROOT_BUFFER_DEFAULT_SIZE 10002 #define GC_ROOT_BUFFER_SIZE_INCREMENT 4000 // TODO Arbitrary number @@ -179,6 +180,8 @@ static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root) GC_PREV_BUF(root)->next = root->next; root->prev = GC_G(unused); GC_G(unused) = GC_TO_ADDR(root); + + GC_G(num_roots)--; GC_BENCH_DEC(root_buf_length); } @@ -196,10 +199,11 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) gc_globals->gc_active = 0; gc_globals->buf = GC_DUMMY_BUF(); + gc_globals->buf_size = GC_DUMMY_BUF_SIZE; + gc_globals->num_roots = 0; gc_globals->unused = GC_INVALID; gc_globals->first_unused = GC_FIRST_REAL_ROOT; - gc_globals->buf_size = GC_DUMMY_BUF_SIZE; gc_globals->next_to_free = GC_INVALID; gc_globals->gc_runs = 0; @@ -298,6 +302,21 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) GC_BENCH_INC(zval_possible_root); + /* TODO It would be more elegant to move this to the end and avoid all the + * special cases here. I'm keeping it for now to preserve exact collection point. */ + if (UNEXPECTED(GC_G(num_roots) >= GC_COLLECTION_THRESHOLD && GC_G(gc_enabled))) { + GC_ADDREF(ref); + gc_collect_cycles(); + GC_DELREF(ref); + if (UNEXPECTED(GC_REFCOUNT(ref)) == 0) { + zval_dtor_func(ref); + return; + } + if (UNEXPECTED(GC_ADDRESS(ref))) { + return; + } + } + newRootAddr = GC_G(unused); if (newRootAddr != GC_INVALID) { newRoot = GC_TO_BUF(newRootAddr); @@ -311,6 +330,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) /* TODO Resize here */ return; } + ZEND_ASSERT(0); GC_ADDREF(ref); gc_collect_cycles(); GC_DELREF(ref); @@ -350,6 +370,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) roots->next = newRootAddr; } + GC_G(num_roots)++; GC_BENCH_INC(zval_buffered); GC_BENCH_INC(root_buf_length); GC_BENCH_PEAK(root_buf_peak, root_buf_length); @@ -739,6 +760,8 @@ static void gc_add_garbage(zend_refcounted *ref) buf->prev = GC_ROOTS_SENTINEL; GC_NEXT_BUF(roots)->prev = addr; roots->next = addr; + + GC_G(num_roots)++; } static int gc_collect_white(zend_refcounted *ref, uint32_t *flags) @@ -1159,6 +1182,8 @@ ZEND_API int zend_gc_collect_cycles(void) p = current->ref; current->prev = GC_G(unused); GC_G(unused) = GC_TO_ADDR(current); + GC_G(num_roots)--; + efree(p); current = next; } diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index 3acf42e9d8f49..730cc016919b6 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -54,6 +54,7 @@ typedef struct _zend_gc_globals { gc_root_buffer *buf; /* preallocated arrays of buffers */ uint32_t buf_size; /* size of the GC buffer */ + uint32_t num_roots; /* number of roots in GC buffer */ uint32_t unused; /* linked list of unused buffers */ uint32_t first_unused; /* first unused buffer */ uint32_t next_to_free; /* next to free in to_free list */ From 948cbd278f74ceebb5d3749c39cce7562c3f4965 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 19 Jan 2018 21:15:03 +0100 Subject: [PATCH 08/10] Forbid nested GC runs During the initial root collection phase the GC is marked as "protected". During this phase addition of new roots is prohibited. Otherwise, while the GC is active, addition of new roots is allowed, but a nested GC is prohibited. GC will be delayed until the current one finishes. --- Zend/tests/gc_016.phpt | 4 ++- Zend/zend_gc.c | 72 ++++++++++++------------------------------ Zend/zend_gc.h | 4 +-- 3 files changed, 25 insertions(+), 55 deletions(-) diff --git a/Zend/tests/gc_016.phpt b/Zend/tests/gc_016.phpt index f1e14dab1a8ce..f082d60973e88 100644 --- a/Zend/tests/gc_016.phpt +++ b/Zend/tests/gc_016.phpt @@ -18,9 +18,11 @@ $a = new Foo(); $a->a = $a; unset($a); var_dump(gc_collect_cycles()); +var_dump(gc_collect_cycles()); echo "ok\n" ?> --EXPECT-- --> int(1) +-> int(0) +int(1) int(1) ok diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index 6895ece142113..d9a8ef4b984d1 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -197,6 +197,7 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) { gc_globals->gc_enabled = 0; gc_globals->gc_active = 0; + gc_globals->gc_protected = 0; gc_globals->buf = GC_DUMMY_BUF(); gc_globals->buf_size = GC_DUMMY_BUF_SIZE; @@ -239,7 +240,6 @@ ZEND_API void gc_reset(void) { GC_G(gc_runs) = 0; GC_G(collected) = 0; - GC_G(gc_full) = 0; #if GC_BENCH GC_G(root_buf_length) = 0; @@ -292,7 +292,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) uint32_t newRootAddr; gc_root_buffer *newRoot; - if (UNEXPECTED(CG(unclean_shutdown)) || UNEXPECTED(GC_G(gc_active))) { + if (UNEXPECTED(CG(unclean_shutdown)) || UNEXPECTED(GC_G(gc_protected))) { return; } @@ -304,7 +304,8 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) /* TODO It would be more elegant to move this to the end and avoid all the * special cases here. I'm keeping it for now to preserve exact collection point. */ - if (UNEXPECTED(GC_G(num_roots) >= GC_COLLECTION_THRESHOLD && GC_G(gc_enabled))) { + if (UNEXPECTED(GC_G(num_roots) >= GC_COLLECTION_THRESHOLD + && GC_G(gc_enabled) && !GC_G(gc_active))) { GC_ADDREF(ref); gc_collect_cycles(); GC_DELREF(ref); @@ -330,31 +331,11 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) /* TODO Resize here */ return; } - ZEND_ASSERT(0); - GC_ADDREF(ref); - gc_collect_cycles(); - GC_DELREF(ref); - if (UNEXPECTED(GC_REFCOUNT(ref)) == 0) { - zval_dtor_func(ref); - return; - } - if (UNEXPECTED(GC_ADDRESS(ref))) { - return; - } - newRootAddr = GC_G(unused); - if (newRootAddr == GC_INVALID) { - /* TODO Resize here */ -#if ZEND_GC_DEBUG - if (!GC_G(gc_full)) { - fprintf(stderr, "GC: no space to record new root candidate\n"); - GC_G(gc_full) = 1; - } -#endif - return; - } + gc_grow_root_buffer(); + newRootAddr = GC_G(first_unused); newRoot = GC_TO_BUF(newRootAddr); - GC_G(unused) = newRoot->prev; + GC_G(first_unused)++; } GC_TRACE_SET_COLOR(ref, GC_PURPLE); @@ -1051,9 +1032,6 @@ ZEND_API int zend_gc_collect_cycles(void) zend_refcounted *p; gc_root_buffer *to_free; uint32_t gc_flags = 0; -#if ZEND_GC_DEBUG - zend_bool orig_gc_full; -#endif if (GC_G(gc_active)) { return 0; @@ -1062,41 +1040,32 @@ ZEND_API int zend_gc_collect_cycles(void) GC_TRACE("Collecting cycles"); GC_G(gc_runs)++; GC_G(gc_active) = 1; + GC_G(gc_protected) = 1; GC_TRACE("Marking roots"); gc_mark_roots(); GC_TRACE("Scanning roots"); gc_scan_roots(); -#if ZEND_GC_DEBUG - orig_gc_full = GC_G(gc_full); - GC_G(gc_full) = 0; -#endif - GC_TRACE("Collecting roots"); count = gc_collect_roots(&gc_flags); -#if ZEND_GC_DEBUG - GC_G(gc_full) = orig_gc_full; -#endif - /* TODO: Handle nested GC */ - GC_G(gc_active) = 0; + GC_G(gc_protected) = 0; to_free = GC_TO_FREE(); if (to_free->next == GC_TO_FREE_SENTINEL) { /* nothing to free */ GC_TRACE("Nothing to free"); + GC_G(gc_active) = 0; return 0; } -#if ZEND_GC_DEBUG - orig_gc_full = GC_G(gc_full); - GC_G(gc_full) = 0; -#endif - if (gc_flags & GC_HAS_DESTRUCTORS) { GC_TRACE("Calling destructors"); + /* The root buffer might be reallocated during destructors calls, + * make sure to reload pointers as necessary. */ + /* Remember reference counters before calling destructors */ current = GC_NEXT_BUF(to_free); while (current != to_free) { @@ -1106,7 +1075,7 @@ ZEND_API int zend_gc_collect_cycles(void) /* Call destructors */ current = GC_NEXT_BUF(to_free); - while (current != to_free) { + while (current != GC_TO_FREE()) { p = current->ref; GC_G(next_to_free) = current->next; if (GC_TYPE(p) == IS_OBJECT) { @@ -1124,23 +1093,24 @@ ZEND_API int zend_gc_collect_cycles(void) } } } - current = GC_NEXT_BUF(current); + current = GC_TO_BUF(GC_G(next_to_free)); } /* Remove values captured in destructors */ current = GC_NEXT_BUF(to_free); - while (current != to_free) { + while (current != GC_TO_FREE()) { GC_G(next_to_free) = current->next; if (GC_REFCOUNT(current->ref) > current->refcount) { gc_remove_nested_data_from_buffer(current->ref, current); } - current = GC_NEXT_BUF(current); + current = GC_TO_BUF(GC_G(next_to_free)); } } /* Destroy zvals */ GC_TRACE("Destroying zvals"); - GC_G(gc_active) = 1; + GC_G(gc_protected) = 1; + to_free = GC_TO_FREE(); current = GC_NEXT_BUF(to_free); while (current != to_free) { p = current->ref; @@ -1194,9 +1164,7 @@ ZEND_API int zend_gc_collect_cycles(void) GC_TRACE("Collection finished"); GC_G(collected) += count; GC_G(next_to_free) = GC_INVALID; -#if ZEND_GC_DEBUG - GC_G(gc_full) = orig_gc_full; -#endif + GC_G(gc_protected) = 0; GC_G(gc_active) = 0; } diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index 730cc016919b6..0ffb7cd3c5cb8 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -49,8 +49,8 @@ typedef struct _gc_root_buffer { typedef struct _zend_gc_globals { zend_bool gc_enabled; - zend_bool gc_active; - zend_bool gc_full; + zend_bool gc_active; /* GC currently running, forbid nested GC */ + zend_bool gc_protected; /* GC collecting roots, forbid root additions */ gc_root_buffer *buf; /* preallocated arrays of buffers */ uint32_t buf_size; /* size of the GC buffer */ From 519ad4c2d1ede7bff999b3c999e769716d181add Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 19 Jan 2018 21:44:52 +0100 Subject: [PATCH 09/10] Ensure roots are collects during temporary gc_disable() Only don't collect roots if gc.enabled=0 at startup, in which case the root buffer is not allocated at all. --- Zend/zend_gc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index d9a8ef4b984d1..d2ffc9727160a 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -327,8 +327,9 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) newRoot = GC_TO_BUF(newRootAddr); GC_G(first_unused)++; } else { - if (!GC_G(gc_enabled)) { - /* TODO Resize here */ + if (GC_G(buf) == GC_DUMMY_BUF()) { + /* This means that the GC is completely disabled, rather than just disabled + * temporarily at run-time. In this case we just let things leak. */ return; } From 70aec4ca608ed2f1bc22c40974cdb413e36132b2 Mon Sep 17 00:00:00 2001 From: Nikita Popov Date: Fri, 19 Jan 2018 21:57:38 +0100 Subject: [PATCH 10/10] Implement very simply dynamic GC buffer resizing --- Zend/zend_gc.c | 36 +++++++++++++++++++++++------------- Zend/zend_gc.h | 5 +++-- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index d2ffc9727160a..6dda6e72173c1 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -72,10 +72,6 @@ #include "zend.h" #include "zend_API.h" -#define GC_COLLECTION_THRESHOLD 10000 -#define GC_ROOT_BUFFER_DEFAULT_SIZE 10002 -#define GC_ROOT_BUFFER_SIZE_INCREMENT 4000 // TODO Arbitrary number - #define GC_HAS_DESTRUCTORS (1<<0) #ifndef ZEND_GC_DEBUG @@ -111,6 +107,10 @@ ZEND_API int (*gc_collect_cycles)(void); #define GC_FIRST_REAL_ROOT ((uint32_t) 2) #define GC_DUMMY_BUF_SIZE GC_FIRST_REAL_ROOT +#define GC_DEFAULT_COLLECTION_THRESHOLD 10000 +#define GC_ROOT_BUFFER_DEFAULT_SIZE \ + (GC_DEFAULT_COLLECTION_THRESHOLD + GC_FIRST_REAL_ROOT) + #define GC_TO_BUF(addr) (GC_G(buf) + (addr)) #define GC_TO_ADDR(buffer) ((buffer) - GC_G(buf)) @@ -201,6 +201,7 @@ static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) gc_globals->buf = GC_DUMMY_BUF(); gc_globals->buf_size = GC_DUMMY_BUF_SIZE; + gc_globals->gc_threshold = GC_DEFAULT_COLLECTION_THRESHOLD; gc_globals->num_roots = 0; gc_globals->unused = GC_INVALID; @@ -275,18 +276,13 @@ ZEND_API void gc_init(void) } } -static void gc_resize_root_buffer(size_t new_size) { - /* Only allow increasing for now */ - ZEND_ASSERT(new_size > GC_G(buf_size)); +static void gc_grow_root_buffer() { + /* Double the buffer size, taking into account the reserved roots */ + size_t new_size = (GC_G(buf_size) - GC_FIRST_REAL_ROOT) * 2 + GC_FIRST_REAL_ROOT; GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1); - GC_G(buf) = realloc(GC_G(buf), sizeof(gc_root_buffer) * new_size); GC_G(buf_size) = new_size; } -static void gc_grow_root_buffer() { - gc_resize_root_buffer(GC_G(buf_size) + GC_ROOT_BUFFER_SIZE_INCREMENT); -} - ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) { uint32_t newRootAddr; @@ -304,7 +300,7 @@ ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) /* TODO It would be more elegant to move this to the end and avoid all the * special cases here. I'm keeping it for now to preserve exact collection point. */ - if (UNEXPECTED(GC_G(num_roots) >= GC_COLLECTION_THRESHOLD + if (UNEXPECTED(GC_G(num_roots) >= GC_G(gc_threshold) && GC_G(gc_enabled) && !GC_G(gc_active))) { GC_ADDREF(ref); gc_collect_cycles(); @@ -1023,6 +1019,17 @@ static void gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buff } } +static void gc_adjust_threshold(int count) { + /* TODO Very simple heuristic for dynamic GC buffer resizing: + * If there are "too few" collections, increase the collection threshold + * by a factor of two. */ + if (count < 100) { + GC_G(gc_threshold) *= 2; + } else if (GC_G(gc_threshold) > GC_DEFAULT_COLLECTION_THRESHOLD) { + GC_G(gc_threshold) /= 2; + } +} + ZEND_API int zend_gc_collect_cycles(void) { int count = 0; @@ -1058,6 +1065,7 @@ ZEND_API int zend_gc_collect_cycles(void) /* nothing to free */ GC_TRACE("Nothing to free"); GC_G(gc_active) = 0; + gc_adjust_threshold(count); return 0; } @@ -1167,6 +1175,8 @@ ZEND_API int zend_gc_collect_cycles(void) GC_G(next_to_free) = GC_INVALID; GC_G(gc_protected) = 0; GC_G(gc_active) = 0; + + gc_adjust_threshold(count); } return count; diff --git a/Zend/zend_gc.h b/Zend/zend_gc.h index 0ffb7cd3c5cb8..c934ab64b0be5 100644 --- a/Zend/zend_gc.h +++ b/Zend/zend_gc.h @@ -42,7 +42,7 @@ typedef struct _gc_root_buffer { zend_refcounted *ref; - uint32_t next; /* double-linked list */ + uint32_t next; uint32_t prev; uint32_t refcount; } gc_root_buffer; @@ -50,7 +50,7 @@ typedef struct _gc_root_buffer { typedef struct _zend_gc_globals { zend_bool gc_enabled; zend_bool gc_active; /* GC currently running, forbid nested GC */ - zend_bool gc_protected; /* GC collecting roots, forbid root additions */ + zend_bool gc_protected; /* GC protected, forbid root additions */ gc_root_buffer *buf; /* preallocated arrays of buffers */ uint32_t buf_size; /* size of the GC buffer */ @@ -58,6 +58,7 @@ typedef struct _zend_gc_globals { uint32_t unused; /* linked list of unused buffers */ uint32_t first_unused; /* first unused buffer */ uint32_t next_to_free; /* next to free in to_free list */ + uint32_t gc_threshold; /* GC collection threshold */ uint32_t gc_runs; uint32_t collected;