diff --git a/gi/pygobject-object.c b/gi/pygobject-object.c
index ca82ffb9..dd073fe8 100644
--- a/gi/pygobject-object.c
+++ b/gi/pygobject-object.c
@@ -1173,14 +1173,16 @@ pygobject_traverse(PyGObject *self, visitproc visit, void *arg)
         for (tmp = data->closures; tmp != NULL; tmp = tmp->next) {
             PyGClosure *closure = tmp->data;
 
-            if (closure->callback) ret = visit(closure->callback, arg);
-            if (ret != 0) return ret;
+            if (self->obj->ref_count == 1) {
+                if (closure->callback) ret = visit(closure->callback, arg);
+                if (ret != 0) return ret;
 
-            if (closure->extra_args) ret = visit(closure->extra_args, arg);
-            if (ret != 0) return ret;
+                if (closure->extra_args) ret = visit(closure->extra_args, arg);
+                if (ret != 0) return ret;
 
-            if (closure->swap_data) ret = visit(closure->swap_data, arg);
-            if (ret != 0) return ret;
+                if (closure->swap_data) ret = visit(closure->swap_data, arg);
+                if (ret != 0) return ret;
+            }
         }
     }
     return ret;
