Move old-space flag from _gc_prev to _gc_next to support 32 bit machines.

Author: markshannon

Include/internal/pycore_gc.h

@@ -56,31 +56,36 @@ static inline int _PyObject_GC_MAY_BE_TRACKED(PyObject *obj) {
 
 
 /* Bit flags for _gc_prev */
-/* Bit 2 is set when tp_finalize is called */
-#define _PyGC_PREV_MASK_FINALIZED  4
+/* Bit 0 is set when tp_finalize is called */
+#define _PyGC_PREV_MASK_FINALIZED  1
 /* Bit 1 is set when the object is in generation which is GCed currently. */
 #define _PyGC_PREV_MASK_COLLECTING 2
+
 /* Bit 0 is set if the object belongs to old space 1 */
-#define _PyGC_PREV_MASK_OLD_SPACE_1    1
-/* The (N-3) most significant bits contain the real address. */
-#define _PyGC_PREV_SHIFT           3
+#define _PyGC_NEXT_MASK_OLD_SPACE_1    1
+
+/* The (N-2) most significant bits contain the real address. */
+#define _PyGC_PREV_SHIFT           2
 #define _PyGC_PREV_MASK            (((uintptr_t) -1) << _PyGC_PREV_SHIFT)
 
 // Lowest bit of _gc_next is used for flags only in GC.
 // But it is always 0 for normal code.
 static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) {
-    uintptr_t next = gc->_gc_next;
+    uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK;
     return (PyGC_Head*)next;
 }
 static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) {
-    gc->_gc_next = (uintptr_t)next;
+    uintptr_t unext = (uintptr_t)next;
+    assert((unext & ~_PyGC_PREV_MASK) == 0);
+    gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext;
 }
 
 // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags.
 static inline PyGC_Head* _PyGCHead_PREV(PyGC_Head *gc) {
     uintptr_t prev = (gc->_gc_prev & _PyGC_PREV_MASK);
     return (PyGC_Head*)prev;
 }
+
 static inline void _PyGCHead_SET_PREV(PyGC_Head *gc, PyGC_Head *prev) {
     uintptr_t uprev = (uintptr_t)prev;
     assert((uprev & ~_PyGC_PREV_MASK) == 0);

Include/internal/pycore_object.h

@@ -243,7 +243,7 @@ static inline void _PyObject_GC_TRACK(
     _PyGCHead_SET_NEXT(last, gc);
     _PyGCHead_SET_PREV(gc, last);
     _PyGCHead_SET_NEXT(gc, generation0);
-    assert((gc->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1) == 0);
+    assert((gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1) == 0);
     generation0->_gc_prev = (uintptr_t)gc;
 }
 

Modules/gcmodule.c

@@ -68,7 +68,7 @@ module gc
 // move_legacy_finalizers() removes this flag instead.
 // Between them, unreachable list is not normal list and we can not use
 // most gc_list_* functions for it.
-#define NEXT_MASK_UNREACHABLE  (1)
+#define NEXT_MASK_UNREACHABLE  (2)
 
 #define AS_GC(op) _Py_AS_GC(op)
 #define FROM_GC(gc) _Py_FROM_GC(gc)
@@ -102,7 +102,7 @@ gc_set_refs(PyGC_Head *g, Py_ssize_t refs)
 static inline void
 gc_reset_refs(PyGC_Head *g, Py_ssize_t refs)
 {
-    g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK)
+    g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED)
         | PREV_MASK_COLLECTING
         | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT);
 }
@@ -119,26 +119,27 @@ gc_decref(PyGC_Head *g)
 static inline uintptr_t
 gc_old_space(PyGC_Head *g)
 {
-    return g->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1;
+    return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1;
 }
 
 static inline uintptr_t
 flip_old_space(uintptr_t space)
 {
-    return space ^ _PyGC_PREV_MASK_OLD_SPACE_1;
+    return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1;
 }
 
 static inline void
 gc_flip_old_space(PyGC_Head *g)
 {
-    g->_gc_prev ^= _PyGC_PREV_MASK_OLD_SPACE_1;
+    g->_gc_next ^= _PyGC_NEXT_MASK_OLD_SPACE_1;
 }
 
 static inline void
 gc_set_old_space(PyGC_Head *g, uintptr_t space)
 {
-    g->_gc_prev &= ~_PyGC_PREV_MASK_OLD_SPACE_1;
-    g->_gc_prev |= space;
+    assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1);
+    g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1;
+    g->_gc_next |= space;
 }
 
 /* set for debugging information */
@@ -286,6 +287,7 @@ gc_list_is_empty(PyGC_Head *list)
 static inline void
 gc_list_append(PyGC_Head *node, PyGC_Head *list)
 {
+    assert((list->_gc_prev & ~_PyGC_PREV_MASK) == 0);
     PyGC_Head *last = (PyGC_Head *)list->_gc_prev;
 
     // last <-> node
@@ -434,7 +436,7 @@ validate_list(PyGC_Head *head, enum flagstates flags)
     PyGC_Head *gc = GC_NEXT(head);
     while (gc != head) {
         PyGC_Head *trueprev = GC_PREV(gc);
-        PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next  & ~NEXT_MASK_UNREACHABLE);
+        PyGC_Head *truenext = GC_NEXT(gc);
         assert(truenext != NULL);
         assert(trueprev == prev);
         assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value);
@@ -458,7 +460,7 @@ validate_old(GCState *gcstate)
         PyGC_Head *gc = GC_NEXT(head);
         while (gc != head) {
             PyGC_Head *trueprev = GC_PREV(gc);
-            PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next  & ~NEXT_MASK_UNREACHABLE);
+            PyGC_Head *truenext = GC_NEXT(gc);
             assert(truenext != NULL);
             assert(trueprev == prev);
             assert(gc_old_space(gc) == space);
@@ -477,7 +479,7 @@ validate_list_is_aging(PyGC_Head *head)
     PyGC_Head *gc = GC_NEXT(head);
     uintptr_t aging_flag = gcstate->aging_space;
     while (gc != head) {
-        PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next  & ~NEXT_MASK_UNREACHABLE);
+        PyGC_Head *truenext = GC_NEXT(gc);
         assert(truenext != NULL);
         assert(gc_old_space(gc) == aging_flag);
         prev = gc;
@@ -611,12 +613,13 @@ visit_reachable(PyObject *op, PyGC_Head *reachable)
         // Manually unlink gc from unreachable list because the list functions
         // don't work right in the presence of NEXT_MASK_UNREACHABLE flags.
         PyGC_Head *prev = GC_PREV(gc);
-        PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE);
+        PyGC_Head *next = GC_NEXT(gc);
         _PyObject_ASSERT(FROM_GC(prev),
                          prev->_gc_next & NEXT_MASK_UNREACHABLE);
         _PyObject_ASSERT(FROM_GC(next),
                          next->_gc_next & NEXT_MASK_UNREACHABLE);
         prev->_gc_next = gc->_gc_next;  // copy NEXT_MASK_UNREACHABLE
+        gc->_gc_next &= ~NEXT_MASK_UNREACHABLE;
         _PyGCHead_SET_PREV(next, prev);
 
         gc_list_append(gc, reachable);
@@ -658,7 +661,6 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable)
     // previous elem in the young list, used for restore gc_prev.
     PyGC_Head *prev = young;
     PyGC_Head *gc = GC_NEXT(young);
-
     /* Invariants:  all objects "to the left" of us in young are reachable
      * (directly or indirectly) from outside the young list as it was at entry.
      *
@@ -718,12 +720,12 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable)
             gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable);
             unreachable->_gc_prev = (uintptr_t)gc;
         }
-        gc = (PyGC_Head*)prev->_gc_next;
+        gc = _PyGCHead_NEXT(prev);
     }
     // young->_gc_prev must be last element remained in the list.
     young->_gc_prev = (uintptr_t)prev;
     // don't let the pollution of the list head's next pointer leak
-    unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE;
+    unreachable->_gc_next &= _PyGC_PREV_MASK;
 }
 
 static void
@@ -780,7 +782,7 @@ move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers)
         PyObject *op = FROM_GC(gc);
 
         _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE);
-        gc->_gc_next &= ~NEXT_MASK_UNREACHABLE;
+        gc->_gc_next &= _PyGC_PREV_MASK;
         next = (PyGC_Head*)gc->_gc_next;
 
         if (has_legacy_finalizer(op)) {
@@ -800,7 +802,7 @@ clear_unreachable_mask(PyGC_Head *unreachable)
     assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0);
     for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) {
         _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE);
-        gc->_gc_next &= ~NEXT_MASK_UNREACHABLE;
+        gc->_gc_next &= _PyGC_PREV_MASK;
         next = (PyGC_Head*)gc->_gc_next;
     }
     validate_list(unreachable, collecting_set_unreachable_clear);
@@ -1525,6 +1527,7 @@ gc_collect_region(PyThreadState *tstate,
     assert(gcstate->garbage != NULL);
     assert(!_PyErr_Occurred(tstate));
 
+    validate_list(from, collecting_clear_unreachable_clear);
     validate_list(to, collecting_clear_unreachable_clear);
 
     deduce_unreachable(from, &unreachable);