gh-98724: Fix the Py_CLEAR() macro in the limited C API

If _Py_TYPEOF() is not available, the limited C API now implements the Py_CLEAR() macro as a function call which hides implementation details. The function uses memcpy() of <string.h> which is not included by <Python.h>. * Add private _Py_Clear() function. * Add _Py_Clear() to Misc/stable_abi.toml.

Author: vstinner

Include/object.h

@@ -612,7 +612,13 @@ static inline void Py_DECREF(PyObject *op)
  * and so avoid type punning. Otherwise, use memcpy() which causes type erasure
  * and so prevents the compiler to reuse an old cached 'op' value after
  * Py_CLEAR().
+ *
+ * If _Py_TYPEOF() is not available, the limited C API implementation uses a
+ * function call to hide implementation details. The function uses memcpy() of
+ * <string.h> which is not included by <Python.h>.
  */
+PyAPI_FUNC(void) _Py_Clear(PyObject **pobj);
+
 #ifdef _Py_TYPEOF
 #define Py_CLEAR(op) \
  do { \
@@ -623,6 +629,8 @@ static inline void Py_DECREF(PyObject *op)
  Py_DECREF(_tmp_old_op); \
  } \
  } while (0)
+#elif defined(Py_LIMITED_API)
+#define Py_CLEAR(op) _Py_Clear(_Py_CAST(PyObject**, &(op)))
 #else
 #define Py_CLEAR(op) \
  do { \

Lib/test/test_stable_abi_ctypes.py

@@ -2386,3 +2386,7 @@
  added = '3.12' # Before 3.12, available in "structmember.h" w/o Py_ prefix
 [const.Py_AUDIT_READ]
  added = '3.12' # Before 3.12, available in "structmember.h"
+
+[function._Py_Clear]
+ added = '3.12'
+ abi_only = true

Misc/stable_abi.toml

@@ -2593,27 +2593,53 @@ test_set_type_size(PyObject *self, PyObject *Py_UNUSED(ignored))
 static PyObject*
 test_py_clear(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
+ PyObject *list = PyList_New(0);
+ if (list == NULL) {
+ return NULL;
+ }
+ assert(Py_REFCNT(list) == 1);
+
  // simple case with a variable
- PyObject *obj = PyList_New(0);
+ PyObject *obj = Py_NewRef(list);
  if (obj == NULL) {
- return NULL;
+ goto error;
  }
+ assert(Py_REFCNT(list) == 2);
  Py_CLEAR(obj);
+ assert(Py_REFCNT(list) == 1);
  assert(obj == NULL);
 
+ // test _Py_Clear() used by the limited C API
+ PyObject *obj2 = Py_NewRef(list);
+ if (obj2 == NULL) {
+ goto error;
+ }
+ assert(Py_REFCNT(list) == 2);
+ _Py_Clear(&obj2);
+ assert(Py_REFCNT(list) == 1);
+ assert(obj2 == NULL);
+
  // gh-98724: complex case, Py_CLEAR() argument has a side effect
  PyObject* array[1];
- array[0] = PyList_New(0);
+ array[0] = Py_NewRef(list);
  if (array[0] == NULL) {
- return NULL;
+ goto error;
  }
 
  PyObject **p = array;
+ assert(Py_REFCNT(list) == 2);
  Py_CLEAR(*p++);
+ assert(Py_REFCNT(list) == 1);
  assert(array[0] == NULL);
  assert(p == array + 1);
 
+ assert(Py_REFCNT(list) == 1);
+ Py_DECREF(list);
  Py_RETURN_NONE;
+
+error:
+ Py_DECREF(list);
+ return NULL;
 }
 
 

Modules/_testcapimodule.c

@@ -2446,6 +2446,23 @@ int Py_IsFalse(PyObject *x)
  return Py_Is(x, Py_False);
 }
 
+void _Py_Clear(PyObject **pobj)
+{
+ PyObject *old_obj = *pobj;
+ if (old_obj == NULL) {
+ return;
+ }
+
+ // gh-99701: In the limited C API, the Py_CLEAR(obj) macro has a type
+ // punning issue, it casts '&obj' to PyObject** to call _Py_Clear(). Use
+ // memcpy() instead of a simple assignment to cause type erasure and so
+ // prevent the compiler to reuse an old cached 'obj' value after
+ // Py_CLEAR().
+ PyObject *null_ptr = NULL;
+ memcpy(pobj, &null_ptr, sizeof(PyObject*));
+ Py_DECREF(old_obj);
+}
+
 #ifdef __cplusplus
 }
 #endif