bpo-46939: Specialize calls to Python classes

Include/cpython/object.h

@@ -234,6 +234,7 @@ struct _typeobject {
  * by code other than the specializer and interpreter. */
 struct _specialization_cache {
  PyObject *getitem;
+ PyObject *init;
 };
 
 /* The *real* layout of a type object when allocated on the heap */

Include/internal/pycore_frame.h

@@ -61,6 +61,7 @@ typedef struct _PyInterpreterFrame {
  PyFrameState f_state; /* What state the frame is in */
  bool is_entry; // Whether this is the "root" frame for the current _PyCFrame.
  bool is_generator;
+ PyObject *self; /* Borrowed reference used by __init__ frames to return self in RETURN_VALUE */
  PyObject *localsplus[1];
 } _PyInterpreterFrame;
 
@@ -118,6 +119,7 @@ _PyFrame_InitializeSpecials(
  frame->f_state = FRAME_CREATED;
  frame->is_entry = false;
  frame->is_generator = false;
+ frame->self = NULL;
 }
 
 /* Gets the pointer to the locals array

Include/internal/pycore_typeobject.h

@@ -43,6 +43,8 @@ extern PyStatus _PyTypes_InitSlotDefs(void);
 
 extern void _PyStaticType_Dealloc(PyTypeObject *type);
 
+extern PyObject *_PyObject_New_Vector(PyTypeObject *type,
+ PyObject *const *args, Py_ssize_t nargs, PyObject *kwds);
 
 #ifdef __cplusplus
 }

Lib/opcode.py

@@ -285,6 +285,7 @@ def jabs_op(name, op, entries=0):
  "PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST",
  "PRECALL_BOUND_METHOD",
  "PRECALL_PYFUNC",
+ "PRECALL_PY_CLASS",
  "RESUME_QUICK",
  "STORE_ATTR_ADAPTIVE",
  "STORE_ATTR_INSTANCE_VALUE",

Lib/test/test_sys.py

@@ -1397,7 +1397,7 @@ class C(object): pass
  def func():
  return sys._getframe()
  x = func()
- check(x, size('3Pi3c7P2ic??2P'))
+ check(x, size('3Pi3c7P2ic??3P'))
  # function
  def func(): pass
  check(func, size('14Pi'))
@@ -1414,7 +1414,7 @@ def bar(cls):
  check(bar, size('PP'))
  # generator
  def get_gen(): yield 1
- check(get_gen(), size('P2P4P4c7P2ic??P'))
+ check(get_gen(), size('P2P4P4c7P2ic??2P'))
  # iterator
  check(iter('abc'), size('lP'))
  # callable-iterator
@@ -1506,7 +1506,7 @@ def delx(self): del self.__x
  '10P' # PySequenceMethods
  '2P' # PyBufferProcs
  '6P'
- '1P' # Specializer cache
+ '2P' # Specializer cache
  )
  class newstyleclass(object): pass
  # Separate block for PyDictKeysObject with 8 keys and 5 entries

Misc/NEWS.d/next/Core and Builtins/2022-03-06-21-50-36.bpo-46939.nlQEGG.rst

@@ -0,0 +1,2 @@
+Calls to Python classes are now specialized. Creating objects from Python
+classes should now be faster. Patch by Ken Jin.

Objects/typeobject.c

@@ -4514,7 +4514,15 @@ object_init(PyObject *self, PyObject *args, PyObject *kwds)
 static PyObject *
 object_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
- if (excess_args(args, kwds)) {
+ return _PyObject_New_Vector(type, (_PyTuple_CAST(args)->ob_item),
+ PyTuple_GET_SIZE(args), kwds);
+}
+
+PyObject *
+_PyObject_New_Vector(PyTypeObject *type, PyObject *const *args,
+ Py_ssize_t nargs, PyObject *kwds)
+{
+ if (nargs || (kwds && PyDict_Check(kwds) && PyDict_GET_SIZE(kwds))) {
  if (type->tp_new != object_new) {
  PyErr_SetString(PyExc_TypeError,
  "object.__new__() takes exactly one argument (the type to instantiate)");

Python/ceval.c

@@ -1587,6 +1587,11 @@ pop_frame(PyThreadState *tstate, _PyInterpreterFrame *frame)
  */
 typedef struct {
  PyObject *kwnames;
+ /* __init__ is special because while it returns None, we need to return self
+ This tells CALL to pass the current self to the new frame (the __init__ frame).
+ Where it is eventually consumed by RETURN_VALUE.
+ */
+ bool init_pass_self;
 } CallShape;
 
 static inline bool
@@ -1618,6 +1623,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
  _PyCFrame cframe;
  CallShape call_shape;
  call_shape.kwnames = NULL; // Borrowed reference. Reset by CALL instructions.
+ call_shape.init_pass_self = 0;
 
  /* WARNING: Because the _PyCFrame lives on the C stack,
  * but can be accessed from a heap allocated object (tstate)
@@ -2386,6 +2392,18 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 
  TARGET(RETURN_VALUE) {
  PyObject *retval = POP();
+ if (frame->self != NULL) {
+ if (Py_IsNone(retval)) {
+ Py_SETREF(retval, frame->self);
+ frame->self = NULL;
+ }
+ /* We need this to continue raising errors when bad-practice
+ __init__s return their non-None values. This is later
+ caught by the interpreter. */
+ else {
+ Py_CLEAR(frame->self);
+ }
+ }
  assert(EMPTY());
  frame->f_state = FRAME_RETURNED;
  _PyFrame_SetStackPointer(frame, stack_pointer);
@@ -4617,6 +4635,40 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
  DISPATCH();
  }
 
+ TARGET(PRECALL_PY_CLASS) {
+ SpecializedCacheEntry *cache = GET_CACHE();
+ _PyAdaptiveEntry *cache0 = &cache[0].adaptive;
+ _PyCallCache *cache1 = &cache[-1].call;
+ int original_oparg = cache->adaptive.original_oparg;
+ int is_method = (PEEK(original_oparg + 2) != NULL);
+ DEOPT_IF(is_method, PRECALL);
+ PyObject *cls = PEEK(original_oparg + 1);
+ DEOPT_IF(!PyType_Check(cls), PRECALL);
+ PyTypeObject *cls_t = (PyTypeObject *)cls;
+ DEOPT_IF(cls_t->tp_version_tag != cache0->version, PRECALL);
+ assert(cls_t->tp_flags & Py_TPFLAGS_HEAPTYPE);
+ PyObject *init = ((PyHeapTypeObject *)cls_t)->_spec_cache.init;
+ assert(PyFunction_Check(init));
+ DEOPT_IF(((PyFunctionObject *)(init))->func_version != cache1->func_version, PRECALL);
+ DEOPT_IF(cls_t->tp_new != PyBaseObject_Type.tp_new, PRECALL);
+ STAT_INC(PRECALL, hit);
+
+ PyObject *self = _PyObject_New_Vector(cls_t, &PEEK(original_oparg),
+ (Py_ssize_t)original_oparg, call_shape.kwnames);
+ if (self == NULL) {
+ goto error;
+ }
+ Py_INCREF(init);
+ PEEK(original_oparg+1) = self;
+ PEEK(original_oparg+2) = init;
+ Py_DECREF(cls);
+
+ /* For use in RETURN_VALUE later */
+ assert(call_shape.init_pass_self == false);
+ call_shape.init_pass_self = true;
+ DISPATCH();
+ }
+
  TARGET(KW_NAMES) {
  assert(call_shape.kwnames == NULL);
  assert(oparg < PyTuple_GET_SIZE(consts));
@@ -4651,6 +4703,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
  _PyFrame_SetStackPointer(frame, stack_pointer);
  new_frame->previous = frame;
  cframe.current_frame = frame = new_frame;
+ if (call_shape.init_pass_self) {
+ assert(frame->self == NULL);
+ frame->self = Py_NewRef(frame->localsplus[0]);
+ call_shape.init_pass_self = false;
+ }
  CALL_STAT_INC(inlined_py_calls);
  goto start_frame;
  }
@@ -4762,6 +4819,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
  _PyFrame_SetStackPointer(frame, stack_pointer);
  new_frame->previous = frame;
  frame = cframe.current_frame = new_frame;
+ if (call_shape.init_pass_self) {
+ assert(frame->self == NULL);
+ frame->self = Py_NewRef(frame->localsplus[0]);
+ call_shape.init_pass_self = false;
+ }
  goto start_frame;
  }
 
@@ -4803,6 +4865,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
  _PyFrame_SetStackPointer(frame, stack_pointer);
  new_frame->previous = frame;
  frame = cframe.current_frame = new_frame;
+ if (call_shape.init_pass_self) {
+ assert(frame->self == NULL);
+ frame->self = Py_NewRef(frame->localsplus[0]);
+ call_shape.init_pass_self = false;
+ }
  goto start_frame;
  }
 
@@ -5617,6 +5684,7 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
 
 error:
  call_shape.kwnames = NULL;
+ call_shape.init_pass_self = false;
  /* Double-check exception status. */
 #ifdef NDEBUG
  if (!_PyErr_Occurred(tstate)) {
@@ -5658,6 +5726,7 @@ MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)
  assert(STACK_LEVEL() == 0);
  _PyFrame_SetStackPointer(frame, stack_pointer);
  frame->f_state = FRAME_RAISED;
+ Py_CLEAR(frame->self);
  TRACE_FUNCTION_UNWIND();
  DTRACE_FUNCTION_EXIT();
  goto exit_unwind;

Python/specialize.c

@@ -591,15 +591,16 @@ initial_counter_value(void) {
 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17
 #define SPEC_FAIL_CALL_CLASS 18
 #define SPEC_FAIL_CALL_PYTHON_CLASS 19
-#define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20
-#define SPEC_FAIL_CALL_BOUND_METHOD 21
-#define SPEC_FAIL_CALL_STR 22
-#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23
-#define SPEC_FAIL_CALL_CLASS_MUTABLE 24
-#define SPEC_FAIL_CALL_KWNAMES 25
-#define SPEC_FAIL_CALL_METHOD_WRAPPER 26
-#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27
-#define SPEC_FAIL_CALL_PYFUNCTION 28
+#define SPEC_FAIL_CALL_PYTHON_CLASS_NON_PY_INIT 20
+#define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 21
+#define SPEC_FAIL_CALL_BOUND_METHOD 22
+#define SPEC_FAIL_CALL_STR 23
+#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 24
+#define SPEC_FAIL_CALL_CLASS_MUTABLE 25
+#define SPEC_FAIL_CALL_KWNAMES 26
+#define SPEC_FAIL_CALL_METHOD_WRAPPER 27
+#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 28
+#define SPEC_FAIL_CALL_PYFUNCTION 29
 
 /* COMPARE_OP */
 #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
@@ -1523,7 +1524,33 @@ specialize_class_call(
  assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
  PyTypeObject *tp = _PyType_CAST(callable);
  if (tp->tp_new == PyBaseObject_Type.tp_new) {
- SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_PYTHON_CLASS);
+ _PyAdaptiveEntry *cache0 = &cache[0].adaptive;
+ _PyCallCache *cache1 = &cache[-1].call;
+ PyObject *descriptor = _PyType_Lookup(tp, &_Py_ID(__init__));
+ if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
+ if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
+ return -1;
+ }
+ PyFunctionObject *func = (PyFunctionObject *)descriptor;
+ PyCodeObject *fcode = (PyCodeObject *)func->func_code;
+ int kind = function_kind(fcode);
+ if (kind != SIMPLE_FUNCTION) {
+ SPECIALIZATION_FAIL(PRECALL, kind);
+ return -1;
+ }
+ assert(tp->tp_version_tag != 0);
+ cache0->version = tp->tp_version_tag;
+ int version = _PyFunction_GetVersionForCurrentState(func);
+ if (version == 0 || version != (uint16_t)version) {
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_OUT_OF_VERSIONS);
+ return -1;
+ }
+ cache1->func_version = version;
+ ((PyHeapTypeObject *)tp)->_spec_cache.init = descriptor;
+ *instr = _Py_MAKECODEUNIT(PRECALL_PY_CLASS, _Py_OPARG(*instr));
+ return 0;
+ }
+ SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_PYTHON_CLASS_NON_PY_INIT);
  return -1;
  }
  if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {