Hide old space flag as much as is practical.

Author: markshannon

Modules/gcmodule.c

@@ -116,6 +116,31 @@ gc_decref(PyGC_Head *g)
     g->_gc_prev -= 1 << _PyGC_PREV_SHIFT;
 }
 
+static inline uintptr_t
+gc_old_space(PyGC_Head *g)
+{
+    return g->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1;
+}
+
+static inline uintptr_t
+flip_old_space(uintptr_t space)
+{
+    return space ^ _PyGC_PREV_MASK_OLD_SPACE_1;
+}
+
+static inline void
+gc_flip_old_space(PyGC_Head *g)
+{
+    g->_gc_prev ^= _PyGC_PREV_MASK_OLD_SPACE_1;
+}
+
+static inline void
+gc_set_old_space(PyGC_Head *g, uintptr_t space)
+{
+    g->_gc_prev &= ~_PyGC_PREV_MASK_OLD_SPACE_1;
+    g->_gc_prev |= space;
+}
+
 /* set for debugging information */
 #define DEBUG_STATS             (1<<0) /* print collection statistics */
 #define DEBUG_COLLECTABLE       (1<<1) /* print collectable objects */
@@ -427,16 +452,16 @@ validate_list(PyGC_Head *head, enum flagstates flags)
 static void
 validate_old(GCState *gcstate)
 {
-    for (uintptr_t gen = 0; gen < 2; gen++) {
-        PyGC_Head *head = &gcstate->old[gen].head;
+    for (uintptr_t space = 0; space < 2; space++) {
+        PyGC_Head *head = &gcstate->old[space].head;
         PyGC_Head *prev = head;
         PyGC_Head *gc = GC_NEXT(head);
         while (gc != head) {
             PyGC_Head *trueprev = GC_PREV(gc);
             PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next  & ~NEXT_MASK_UNREACHABLE);
             assert(truenext != NULL);
             assert(trueprev == prev);
-            assert((gc->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1) == gen);
+            assert(gc_old_space(gc) == space);
             prev = gc;
             gc = truenext;
         }
@@ -454,7 +479,7 @@ validate_list_is_aging(PyGC_Head *head)
     while (gc != head) {
         PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next  & ~NEXT_MASK_UNREACHABLE);
         assert(truenext != NULL);
-        assert((gc->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1) == aging_flag);
+        assert(gc_old_space(gc) == aging_flag);
         prev = gc;
         gc = truenext;
     }
@@ -1240,25 +1265,13 @@ gc_collect_region(PyThreadState *tstate,
                   int untrack,
                   struct gc_collection_stats *stats);
 
-static Py_ssize_t
-make_space0(PyGC_Head *list)
-{
-    Py_ssize_t size = 0;
-    PyGC_Head *gc;
-    for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) {
-        gc->_gc_prev &= ~_PyGC_PREV_MASK_OLD_SPACE_1;
-        size++;
-    }
-    return size;
-}
-
-static Py_ssize_t
-make_space1(PyGC_Head *list)
+static inline Py_ssize_t
+gc_list_set_space(PyGC_Head *list, uintptr_t space)
 {
     Py_ssize_t size = 0;
     PyGC_Head *gc;
     for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) {
-        gc->_gc_prev |= _PyGC_PREV_MASK_OLD_SPACE_1;
+        gc_set_old_space(gc, space);
         size++;
     }
     return size;
@@ -1285,7 +1298,7 @@ gc_collect_young(PyThreadState *tstate,
          * do that here */
         PyGC_Head *gc;
         for (gc = GC_NEXT(young); gc != young; gc = GC_NEXT(gc)) {
-            gc->_gc_prev |= _PyGC_PREV_MASK_OLD_SPACE_1;
+            gc_set_old_space(gc, 1);
         }
     }
     validate_old(gcstate);
@@ -1296,13 +1309,13 @@ gc_collect_young(PyThreadState *tstate,
     Py_ssize_t surivivor_count = 0;
     if (gcstate->aging_space) {
         /* objects in aging space have bit set, so we set it here */
-        surivivor_count = make_space1(&survivors);
+        surivivor_count = gc_list_set_space(&survivors, 1);
     }
     else {
         PyGC_Head *gc;
         for (gc = GC_NEXT(&survivors); gc != &survivors; gc = GC_NEXT(gc)) {
 #ifdef GC_DEBUG
-            assert((gc->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1) == 0);
+            assert(gc_old_space(gc) == 0);
 #endif
             surivivor_count++;
         }
@@ -1320,21 +1333,19 @@ static void
 gc_collect_aging(PyThreadState *tstate,
                  struct gc_collection_stats *stats)
 {
-    /* For now just collect middle generation */
     GCState *gcstate = &tstate->interp->gc;
+    uintptr_t oldest_space = flip_old_space(gcstate->aging_space);
     PyGC_Head *young = &gcstate->young.head;
     PyGC_Head *aging = &gcstate->old[gcstate->aging_space].head;
-    PyGC_Head *oldest = &gcstate->old[gcstate->aging_space^1].head;
+    PyGC_Head *oldest = &gcstate->old[oldest_space].head;
     /* Need to tag all objects as part of oldest generation */
-    Py_ssize_t size;
     if (gcstate->aging_space) {
         /* No need to tag young objects */
-        size = make_space0(aging);
     }
     else {
-        make_space1(young);
-        size = make_space1(aging);
+        gc_list_set_space(young, 1);
     }
+    Py_ssize_t size = gc_list_set_space(aging, oldest_space);
     /* merge young in pending before collecting */
     gc_list_merge(young, aging);
     gcstate->young.count = 0;
@@ -1348,10 +1359,10 @@ gc_collect_aging(PyThreadState *tstate,
 }
 
 static inline int
-IS_IN_OLDEST(PyGC_Head *head, uintptr_t aging_space)
+IS_IN_OLDEST(PyGC_Head *gc, uintptr_t aging_space)
 {
-    assert(aging_space == 0 || aging_space == _PyGC_PREV_MASK_OLD_SPACE_1);
-    return (head->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1) == (aging_space^_PyGC_PREV_MASK_OLD_SPACE_1);
+    assert(aging_space == 0 || flip_old_space(aging_space) == 0);
+    return gc_old_space(gc) != aging_space;
 }
 
 struct container_and_flag {
@@ -1365,13 +1376,13 @@ visit_add_to_container(PyObject *op, void *arg)
 {
     OBJECT_STAT_INC(object_visits);
     struct container_and_flag *cf = (struct container_and_flag *)arg;
-    uintptr_t oldest_flag = cf->aging_space ^ _PyGC_PREV_MASK_OLD_SPACE_1;
+    uintptr_t aging = cf->aging_space;
     if (_PyObject_IS_GC(op)) {
         PyGC_Head *gc = AS_GC(op);
         if (_PyObject_GC_IS_TRACKED(op) &&
-            (gc->_gc_prev & _PyGC_PREV_MASK_OLD_SPACE_1) == oldest_flag) {
+            gc_old_space(gc) != aging) {
             assert(!_Py_IsImmortal(op));
-            gc->_gc_prev ^= _PyGC_PREV_MASK_OLD_SPACE_1;
+            gc_flip_old_space(gc);
             gc_list_move(gc, cf->container);
         }
     }
@@ -1437,7 +1448,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
         PyGC_Head *tmp = aging;
         aging = oldest;
         oldest = tmp;
-        gcstate->aging_space ^= _PyGC_PREV_MASK_OLD_SPACE_1;
+        gcstate->aging_space = flip_old_space(gcstate->aging_space);
     }
     validate_old(gcstate);
     while (region_size < work_to_do) {
@@ -1447,8 +1458,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
         }
         PyGC_Head *gc = _PyGCHead_NEXT(oldest);
         gc_list_move(gc, &increment);
-        gc->_gc_prev &= ~_PyGC_PREV_MASK_OLD_SPACE_1;
-        gc->_gc_prev |= gcstate->aging_space;
+        gc_set_old_space(gc, gcstate->aging_space);
         region_size += expand_region_transitively_reachable(&increment, gcstate);
     }
     validate_old(gcstate);
@@ -1477,7 +1487,7 @@ gc_collect_full(PyThreadState *tstate,
     gcstate->young.count = 0;
     PyGC_Head *gc = GC_NEXT(old1);
     while (gc != old1) {
-        gc->_gc_prev &= ~_PyGC_PREV_MASK_OLD_SPACE_1;
+        gc_set_old_space(gc, 0);
         gc = GC_NEXT(gc);
     }
     gc_list_merge(old1, old0);
@@ -2137,7 +2147,7 @@ gc_freeze_impl(PyObject *module)
     PyGC_Head*old1 = &gcstate->old[1].head;
     gc_list_merge(old0, &gcstate->permanent_generation.head);
     gcstate->old[0].count = 0;
-    make_space0(old1);
+    gc_list_set_space(old1, 0);
     gc_list_merge(old1, &gcstate->permanent_generation.head);
     gcstate->old[1].count = 0;
     validate_old(gcstate);