gh-111140: Fix edge case in PyLong_AsNativeBytes where large negative longs may require an extra byte

Lib/test/test_capi/test_long.py

@@ -451,11 +451,15 @@ def test_long_asnativebytes(self):
  (MAX_SSIZE, SZ),
  (MAX_USIZE, SZ + 1),
  (-MAX_SSIZE, SZ),
- (-MAX_USIZE, SZ + 1),
+ (-MAX_USIZE, SZ),
  (2**255-1, 32),
  (-(2**255-1), 32),
+ (2**255, 33),
+ (-(2**255), 32), # edge case
  (2**256-1, 33),
- (-(2**256-1), 33),
+ (-(2**256-1), 32), # edge case
+ (2**256, 33),
+ (-(2**256), 33),
  ]:
  with self.subTest(f"sizeof-{v:X}"):
  buffer = bytearray(b"\x5a")
@@ -500,10 +504,10 @@ def test_long_asnativebytes(self):
  (256, b'\x01\x00', 2),
  # Extracts successfully (unsigned), but requests 9 bytes
  (2**63, b'\x80' + b'\x00' * 7, 9),
- # "Extracts", but requests 9 bytes
- (-2**63, b'\x80' + b'\x00' * 7, 9),
  (2**63, b'\x00\x80' + b'\x00' * 7, 9),
- (-2**63, b'\xff\x80' + b'\x00' * 7, 9),
+ # Extracts successfully and only requests 8 bytes
+ (-2**63, b'\x80' + b'\x00' * 7, 8),
+ (-2**63, b'\xff\x80' + b'\x00' * 7, 8),
 
  (2**255-1, b'\x7f' + b'\xff' * 31, 32),
  (-(2**255-1), b'\x80' + b'\x00' * 30 + b'\x01', 32),
@@ -516,9 +520,11 @@ def test_long_asnativebytes(self):
  # into a 32-byte buffer, though negative number may be unrecoverable
  (2**256-1, b'\xff' * 32, 33),
  (2**256-1, b'\x00' + b'\xff' * 32, 33),
- (-(2**256-1), b'\x00' * 31 + b'\x01', 33),
- (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 33),
- (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 33),
+ # Negative 256 bits of integer will only request 32 bytes, since the
+ # top-most bit is the sign bit as well as the magnitude.
+ (-(2**256-1), b'\x00' * 31 + b'\x01', 32),
+ (-(2**256-1), b'\xff' + b'\x00' * 31 + b'\x01', 32),
+ (-(2**256-1), b'\xff\xff' + b'\x00' * 31 + b'\x01', 32),
 
  # The classic "Windows HRESULT as negative number" case
  # HRESULT hr;

Misc/NEWS.d/next/C API/2024-02-28-15-50-01.gh-issue-111140.mpwcUg.rst

@@ -0,0 +1,3 @@
+Allow :c:func:`PyLong_AsNativeBytes` to extract negative numbers requiring
+every single bit of the target buffer into the buffer without requesting a
+larger one.

Objects/longobject.c

@@ -1199,17 +1199,18 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int endianness)
  _PyLong_AsByteArray(v, buffer, (size_t)n, little_endian, 1, 0);
  }
 
- // More efficient calculation for number of bytes required?
+ /* Calculates the number of bits required for the *absolute* value
+ * of v. This does not take sign into account, only magnitude. */
  size_t nb = _PyLong_NumBits((PyObject *)v);
  /* Normally this would be((nb - 1) / 8) + 1 to avoid rounding up
  * multiples of 8 to the next byte, but we add an implied bit for
  * the sign and it cancels out. */
- size_t n_needed = (nb / 8) + 1;
- res = (Py_ssize_t)n_needed;
- if ((size_t)res != n_needed) {
-  PyErr_SetString(PyExc_OverflowError,
-  "value too large to convert");
- res = -1;
+ res = (Py_ssize_t)(nb / 8) + 1;
+ /* The edge case of a negative value where the sign bit is at the
+  * MSB of one byte needs special handling to avoid requesting an
+ * extra byte, even though it could be properly represented. */
+ if (_PyLong_IsNegative(v) && !(nb % 8)) {
+ res -= 1;
  }
  }