From 7cfb6552ffd10068cf75abde2d653b36878cc8d5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2026 11:17:39 +0200 Subject: [PATCH 1/6] gh-148233: Add Py_HashBuffer() to the limited C API (#157966) --- Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.16.rst | 3 +- Include/cpython/pyhash.h | 2 - Include/pyhash.h | 4 ++ Lib/test/test_capi/test_hash.py | 5 ++- Lib/test/test_stable_abi_ctypes.py | 1 + ...-09-22-23-20-53.gh-issue-148233.U_2x-F.rst | 1 + Misc/stable_abi.toml | 2 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/hash.c | 15 ------- Modules/_testlimitedcapi.c | 3 ++ Modules/_testlimitedcapi/hash.c | 40 +++++++++++++++++++ Modules/_testlimitedcapi/parts.h | 1 + PC/python3dll.c | 1 + PCbuild/_testlimitedcapi.vcxproj | 1 + PCbuild/_testlimitedcapi.vcxproj.filters | 1 + 16 files changed, 63 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst create mode 100644 Modules/_testlimitedcapi/hash.c diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 86080fac716383..3fbff9a0d65362 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -969,6 +969,7 @@ func,Py_GetPlatform,3.2,, func,Py_GetRecursionLimit,3.2,, func,Py_GetVersion,3.2,, data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_HashBuffer,3.16,, func,Py_IS_TYPE,3.15,, func,Py_IncRef,3.2,, func,Py_Initialize,3.2,, diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 7e02924babe952..8362b1ef7e312b 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1084,7 +1084,8 @@ C API changes New features ------------ -* TODO +* Add :c:func:`Py_HashBuffer` to the limited C API. + (Contributed by Victor Stinner in :gh:`148233`.) Porting to Python 3.16 ---------------------- diff --git a/Include/cpython/pyhash.h b/Include/cpython/pyhash.h index dac223368dbe5e..39ccffa6f2b90f 100644 --- a/Include/cpython/pyhash.h +++ b/Include/cpython/pyhash.h @@ -50,5 +50,3 @@ _Py_HashPointer(const void *ptr) } PyAPI_FUNC(Py_hash_t) PyObject_GenericHash(PyObject *); - -PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len); diff --git a/Include/pyhash.h b/Include/pyhash.h index 3e23e2758808d7..acadf28f63c5c3 100644 --- a/Include/pyhash.h +++ b/Include/pyhash.h @@ -47,6 +47,10 @@ extern "C" { # endif /* uint64_t && uint32_t && aligned */ #endif /* Py_HASH_ALGORITHM */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 16) +PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len); +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_HASH_H # include "cpython/pyhash.h" diff --git a/Lib/test/test_capi/test_hash.py b/Lib/test/test_capi/test_hash.py index f553ffb0d90f33..d03c3e88087df5 100644 --- a/Lib/test/test_capi/test_hash.py +++ b/Lib/test/test_capi/test_hash.py @@ -1,7 +1,9 @@ import sys import unittest from test.support import import_helper + _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') SIZEOF_VOID_P = _testcapi.SIZEOF_VOID_P @@ -79,7 +81,8 @@ def python_hash_pointer(x): self.assertEqual(hash_pointer(VOID_P_MAX), -2) def test_hash_buffer(self): - hash_buffer = _testcapi.hash_buffer + # Test Py_HashBuffer() + hash_buffer = _testlimitedcapi.hash_buffer def check(data): self.assertEqual(hash_buffer(data), hash(data)) diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 09ee2d53f98f58..aed7451e89b156 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -929,6 +929,7 @@ def test_windows_feature_macros(self): "Py_GetRecursionLimit", "Py_GetVersion", "Py_HasFileSystemDefaultEncoding", + "Py_HashBuffer", "Py_IS_TYPE", "Py_IncRef", "Py_Initialize", diff --git a/Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst b/Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst new file mode 100644 index 00000000000000..069d547a3022b3 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-22-23-20-53.gh-issue-148233.U_2x-F.rst @@ -0,0 +1 @@ +Add :c:func:`Py_HashBuffer` to the limited C API. Patch by Victor Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index d59a7c788fa9e0..36d4f094f9e85a 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2773,6 +2773,8 @@ added = '3.15' [const.Py_tp_module] added = '3.15' +[function.Py_HashBuffer] + added = '3.16' # PEP 757 import/export API. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index d10ed146db92b6..696fec1290e5f3 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -174,7 +174,7 @@ @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tokenizer.c _testinternalcapi/tuple.c _testinternalcapi/typecache.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c _testcapi/marshal.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c _testlimitedcapi/type.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c _testlimitedcapi/type.c _testlimitedcapi/hash.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/hash.c b/Modules/_testcapi/hash.c index 1525344a93fbcf..87d535c585cc27 100644 --- a/Modules/_testcapi/hash.c +++ b/Modules/_testcapi/hash.c @@ -66,20 +66,6 @@ hash_pointer(PyObject *Py_UNUSED(module), PyObject *arg) } -static PyObject * -hash_buffer(PyObject *Py_UNUSED(module), PyObject *args) -{ - char *ptr; - Py_ssize_t len; - if (!PyArg_ParseTuple(args, "y#", &ptr, &len)) { - return NULL; - } - - Py_hash_t hash = Py_HashBuffer(ptr, len); - return long_from_hash(hash); -} - - static PyObject * object_generichash(PyObject *Py_UNUSED(module), PyObject *arg) { @@ -92,7 +78,6 @@ object_generichash(PyObject *Py_UNUSED(module), PyObject *arg) static PyMethodDef test_methods[] = { {"hash_getfuncdef", hash_getfuncdef, METH_NOARGS}, {"hash_pointer", hash_pointer, METH_O}, - {"hash_buffer", hash_buffer, METH_VARARGS}, {"object_generichash", object_generichash, METH_O}, {NULL}, }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 1f7392dd430d4c..b30e32c56c5704 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -91,6 +91,9 @@ module_exec(PyObject *mod) if (_PyTestLimitedCAPI_Init_Type(mod) < 0) { return -1; } + if (_PyTestLimitedCAPI_Init_Hash(mod) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testlimitedcapi/hash.c b/Modules/_testlimitedcapi/hash.c new file mode 100644 index 00000000000000..a17359d5aa45fe --- /dev/null +++ b/Modules/_testlimitedcapi/hash.c @@ -0,0 +1,40 @@ +// Need limited C API version 3.16 for Py_HashBuffer() +#define Py_LIMITED_API 0x03100000 + +#include "parts.h" +#include "util.h" + + +static PyObject* +long_from_hash(Py_hash_t hash) +{ + Py_BUILD_ASSERT(sizeof(long long) >= sizeof(hash)); + return PyLong_FromLongLong(hash); +} + + +// Test Py_HashBuffer() +static PyObject* +hash_buffer(PyObject *Py_UNUSED(module), PyObject *args) +{ + char *ptr; + Py_ssize_t len; + if (!PyArg_ParseTuple(args, "y#", &ptr, &len)) { + return NULL; + } + + Py_hash_t hash = Py_HashBuffer(ptr, len); + return long_from_hash(hash); +} + + +static PyMethodDef test_methods[] = { + {"hash_buffer", hash_buffer, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Hash(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index e34f7dd6809572..e8a2d82fb94aea 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -51,5 +51,6 @@ int _PyTestLimitedCAPI_Init_File(PyObject *module); int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); int _PyTestLimitedCAPI_Init_Run(PyObject *module); int _PyTestLimitedCAPI_Init_Type(PyObject *module); +int _PyTestLimitedCAPI_Init_Hash(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/PC/python3dll.c b/PC/python3dll.c index e0be9d65a93cda..d1b7bafe6abb3c 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -67,6 +67,7 @@ EXPORT_FUNC(Py_GetProgramName) EXPORT_FUNC(Py_GetPythonHome) EXPORT_FUNC(Py_GetRecursionLimit) EXPORT_FUNC(Py_GetVersion) +EXPORT_FUNC(Py_HashBuffer) EXPORT_FUNC(Py_IncRef) EXPORT_FUNC(Py_Initialize) EXPORT_FUNC(Py_InitializeEx) diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index c2bdea923f0f6d..4218e0ed3945d2 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -121,6 +121,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 4c999949474b97..ddef60e599d485 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -37,6 +37,7 @@ + From 71f6c005dc205f68d4435af319142e88a1405a80 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2026 11:20:30 +0200 Subject: [PATCH 2/6] gh-157710: Defer allocation in PyUnicodeWriter_Create() (#157969) PyUnicodeWriter_Create(length) no longer allocates 'length' characters immediately. The allocation of the buffer is now done lazily at the first write, except if the read-only optimization is used. So the read-only optimization can also be used even if length is greater than 0. No longer overallocate the first buffer (at the first write). Only overallocate when the buffer is resized (at the second write). It avoids the need to truncate in PyUnicodeWriter_Finish() when PyUnicodeWriter_Create(length) used the exact output length. Add get_buffer() method to writer tests. --- Lib/test/test_capi/test_unicode.py | 122 +++++++++++++++++++++-------- Modules/_testcapi/unicode.c | 26 ++++++ Objects/unicode_writer.c | 59 +++++++------- 3 files changed, 144 insertions(+), 63 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 9bfb148f87b585..f4bd961017b0ed 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1880,6 +1880,36 @@ def test_basic(self): self.assertEqual(writer.finish(), "var=long value 'repr'") + def test_create(self): + # Test PyUnicodeWriter_Create() with non-zero size + s = 'Monty Python' + + # Preallocate the exact length. Use 2 writes to force the creation + # of a buffer: + # 1. Use the read-only optimization. + # 2. Allocate a buffer of length character. + # No resize needed in finish(). + writer = self.create_writer(len(s)) + writer.write_str(s[:5]) + self.assertEqual(writer.get_buffer(), (5, 127, True)) + writer.write_str(s[5:]) + self.assertEqual(writer.get_buffer(), (len(s), 127, False)) + self.assertEqual(writer.finish(), s) + + # Preallocate len(s)-1 characters. Use 3 writes: + # 1. Use read-only optimization. + # 2. Allocate a buffer of len-1 characters. + # 3. Resize the buffer with overallocation. + # finish() has to truncate the buffer. + writer = self.create_writer(len(s) - 1) + writer.write_str(s[:2]) + self.assertEqual(writer.get_buffer(), (2, 127, True)) + writer.write_str(s[2:5]) + self.assertEqual(writer.get_buffer(), (len(s) - 1, 127, False)) + writer.write_str(s[5:]) + self.assertGreater(writer.get_buffer()[0], len(s)) + self.assertEqual(writer.finish(), s) + def test_repr_null(self): writer = self.create_writer(0) writer.write_utf8(b'var=', -1) @@ -2087,32 +2117,39 @@ def test_substring_empty(self): def test_singletons(self): for size in (0, 123): with self.subTest(size=size): + # PyUnicodeWriter_Finish() returns the empty string singleton + # if no character has been written. writer = self.create_writer(size) writer.write_utf8(b'utf8', 0) writer.write_ascii(b'ascii', 0) writer.write_widechar(b'wstr', 0) writer.write_ucs4(b'ucs4', 0) - writer.write_substring('text', 0, 0) + writer.write_substring('text', 2, 2) + self.assertEqual(writer.get_buffer(), (None, 127, False)) self.assertIs(writer.finish(), '') for size in (0, 123): for ch in range(256): with self.subTest(size=size, ch=ch): ch = chr(ch) + maxchar = (255 if ord(ch) >= 128 else 127) - # If the first write is a Latin1 character and no buffer - # was allocated yet, use the singleton as the read-only - # buffer + # PyUnicodeWriter_WriteChar(ch) uses the read-only + # optimization with the character singleton if ch is a + # Latin1 character and no buffer was allocated yet. writer = self.create_writer(size) writer.write_char(ord(ch)) + self.assertEqual(writer.get_buffer(), + (1, maxchar, True)) self.assertIs(writer.finish(), ch) - # PyUnicodeWriter_Finish() replaces the buffer - # with the singleton + # PyUnicodeWriter_Finish() replaces the buffer with the + # singleton. Use PyUnicodeWriter_WriteSubstring() to avoid + # the read-only buffer optimization. writer = self.create_writer(size) - # Use PyUnicodeWriter_WriteSubstring() to avoid - # the read-only buffer optimization - writer.write_substring(ch + 'xxx', 0, 1) + writer.write_substring('xxx' + ch + 'y', 3, 4) + self.assertEqual(writer.get_buffer(), + (size or 1, maxchar, False)) self.assertIs(writer.finish(), ch) @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)') @@ -2135,7 +2172,8 @@ def test_detect_overflow(self): def test_memory_error(self): # Inject MemoryError in PyUnicodeWriter_WriteStr() writer = self.create_writer(0) - writer.write_str("start") + writer.write_utf8(b"start", -1) + self.assertEqual(writer.get_buffer(), (5, 127, False)) with self.assertRaises(MemoryError): with support.inject_memory_error_cm(): # Resize the internal str object @@ -2143,24 +2181,35 @@ def test_memory_error(self): writer.write_str(" end") self.assertEqual(writer.finish(), "start end") - # Inject MemoryError in PyUnicodeWriter_Finish() + # Inject MemoryError in PyUnicodeWriter_Finish(). Use write_utf8() to + # allocate a buffer of 1024 character. finish() needs to truncate the + # buffer to 3 characters. writer = self.create_writer(1024) - writer.write_str("abc") + writer.write_utf8(b"abc", -1) + self.assertEqual(writer.get_buffer(), (1024, 127, False)) with self.assertRaises(MemoryError): with support.inject_memory_error_cm(): - # Need to truncate the internal str object writer.finish() def test_change_kind(self): writer = self.create_writer(0) + # Create an ASCII buffer writer.write_str('ascii ') + self.assertEqual(writer.get_buffer()[1], 127) + # Change the buffer to UCS1 writer.write_str('latin1:\xe9 ') + self.assertEqual(writer.get_buffer()[1], 255) + # Change the buffer to UCS2 writer.write_str('ucs2:\u20ac ') + self.assertEqual(writer.get_buffer()[1], 0xffff) + # Change the buffer to UCS4 writer.write_str('ucs4:\U0010ffff') + self.assertEqual(writer.get_buffer()[1], 0x10_ffff) + self.assertEqual(writer.finish(), 'ascii latin1:\xe9 ucs2:\u20ac ucs4:\U0010ffff') @@ -2168,27 +2217,38 @@ def test_readonly_optim(self): # Read-only optimization: if the first and only write is a Python str # object and no buffer was allocated yet, return the object unchanged unique_string = 'unique string' - writer = self.create_writer(0) - writer.write_str(unique_string) - self.assertIs(writer.finish(), unique_string) + expected = (len(unique_string), 127, True) + for size in (0, 123): + with self.subTest(size=size): + # PyUnicodeWriter_WriteStr() optimization + writer = self.create_writer(size) + writer.write_str(unique_string) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) - writer = self.create_writer(0) - writer.write_substring(unique_string, 0, len(unique_string)) - self.assertIs(writer.finish(), unique_string) + # PyUnicodeWriter_WriteSubstring() optimization + writer = self.create_writer(size) + writer.write_substring(unique_string, 0, len(unique_string)) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) - class MyStr: - def __str__(self): - return unique_string - writer = self.create_writer(0) - writer.write_str(MyStr()) - self.assertIs(writer.finish(), unique_string) + # PyUnicodeWriter_WriteStr() optimization + class MyStr: + def __str__(self): + return unique_string + writer = self.create_writer(size) + writer.write_str(MyStr()) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) - class MyRepr: - def __repr__(self): - return unique_string - writer = self.create_writer(0) - writer.write_repr(MyRepr()) - self.assertIs(writer.finish(), unique_string) + # PyUnicodeWriter_WriteRepr() optimization + class MyRepr: + def __repr__(self): + return unique_string + writer = self.create_writer(size) + writer.write_repr(MyRepr()) + self.assertEqual(writer.get_buffer(), expected) + self.assertIs(writer.finish(), unique_string) # Test PyUnicodeWriter_Format() diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 8c9e3e9b5321d4..c62813f4f3768d 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -730,6 +730,31 @@ writer_get_pointer(PyObject *self_raw, PyObject *args) } +static PyObject* +writer_get_buffer(PyObject *self_raw, PyObject *args) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + _PyUnicodeWriter *writer = (_PyUnicodeWriter*)self->writer; + PyObject *allocated; + Py_UCS4 maxchar; + if (writer->buffer) { + allocated = PyLong_FromSsize_t(PyUnicode_GET_LENGTH(writer->buffer)); + maxchar = PyUnicode_MAX_CHAR_VALUE(writer->buffer); + } + else { + allocated = Py_None; + maxchar = writer->min_char; + } + return Py_BuildValue("(NkN)", + allocated, (unsigned long)maxchar, + PyBool_FromLong(writer->readonly)); +} + + static PyObject* writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) { @@ -755,6 +780,7 @@ static PyMethodDef writer_methods[] = { {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS}, {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS}, {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, + {"get_buffer", _PyCFunction_CAST(writer_get_buffer), METH_VARARGS}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index fc3a95cd97e421..d6564ce84ed54e 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -151,10 +151,9 @@ PyUnicodeWriter_Create(Py_ssize_t length) _PyUnicodeWriter *writer = (_PyUnicodeWriter *)pub_writer; _PyUnicodeWriter_Init(writer); - if (_PyUnicodeWriter_Prepare(writer, length, 127) < 0) { - PyUnicodeWriter_Discard(pub_writer); - return NULL; - } + // The buffer is created lazily at the first write, except if + // the read-only optimization is used. + writer->min_length = length; writer->overallocate = 1; return pub_writer; @@ -189,9 +188,6 @@ int _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, Py_ssize_t length, Py_UCS4 maxchar) { - Py_ssize_t newlen; - PyObject *newbuffer; - assert(length >= 0); assert(maxchar <= _Py_MAX_UNICODE); @@ -203,50 +199,50 @@ _PyUnicodeWriter_PrepareInternal(_PyUnicodeWriter *writer, PyErr_NoMemory(); return -1; } - newlen = writer->pos + length; + Py_ssize_t alloc = writer->pos + length; maxchar = Py_MAX(maxchar, writer->min_char); + PyObject *newbuffer; if (writer->buffer == NULL) { assert(!writer->readonly); - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { - /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; - } - if (newlen < writer->min_length) - newlen = writer->min_length; + // Do not overallocate at the first allocation, but use min_length + if (alloc < writer->min_length) + alloc = writer->min_length; - writer->buffer = PyUnicode_New(newlen, maxchar); + writer->buffer = PyUnicode_New(alloc, maxchar); if (writer->buffer == NULL) return -1; } - else if (newlen > writer->size) { - if (writer->overallocate - && newlen <= (PY_SSIZE_T_MAX - newlen / OVERALLOCATE_FACTOR)) { + else if (alloc > writer->size) { + // Do not overallocate at the first allocation, but use min_length + int overallocate = (writer->overallocate && !writer->readonly); + if (overallocate + && alloc <= (PY_SSIZE_T_MAX - alloc / OVERALLOCATE_FACTOR)) { /* overallocate to limit the number of realloc() */ - newlen += newlen / OVERALLOCATE_FACTOR; + alloc += alloc / OVERALLOCATE_FACTOR; } - if (newlen < writer->min_length) - newlen = writer->min_length; + if (alloc < writer->min_length) + alloc = writer->min_length; if (maxchar > writer->maxchar || writer->readonly) { /* resize + widen */ maxchar = Py_MAX(maxchar, writer->maxchar); - newbuffer = PyUnicode_New(newlen, maxchar); + newbuffer = PyUnicode_New(alloc, maxchar); if (newbuffer == NULL) return -1; _PyUnicode_FastCopyCharacters(newbuffer, 0, writer->buffer, 0, writer->pos); - Py_DECREF(writer->buffer); writer->readonly = 0; + Py_DECREF(writer->buffer); + writer->buffer = newbuffer; } else { - newbuffer = _PyUnicode_ResizeCompact(writer->buffer, newlen); + newbuffer = _PyUnicode_ResizeCompact(writer->buffer, alloc); if (newbuffer == NULL) return -1; + writer->buffer = newbuffer; } - writer->buffer = newbuffer; } else if (maxchar > writer->maxchar) { assert(!writer->readonly); @@ -310,13 +306,12 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) { assert(PyUnicode_Check(str)); - Py_UCS4 maxchar; - Py_ssize_t len; - - len = PyUnicode_GET_LENGTH(str); - if (len == 0) + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (len == 0) { return 0; - maxchar = PyUnicode_MAX_CHAR_VALUE(str); + } + Py_UCS4 maxchar = PyUnicode_MAX_CHAR_VALUE(str); + if (maxchar > writer->maxchar || len > writer->size - writer->pos) { if (writer->buffer == NULL) { assert(_PyUnicode_CheckConsistency(str, 1)); From 55ecc0a40918ba0346bc83ef639b655b8a883645 Mon Sep 17 00:00:00 2001 From: stratakis Date: Wed, 23 Sep 2026 12:01:48 +0200 Subject: [PATCH 3/6] gh-156722, test_dtrace: Terminate tracing processes gracefully on timeout (#156753) Send SIGTERM to the tracing process group before falling back to SIGKILL. --- Lib/test/test_dtrace.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 126e47a20cb7c6..946f6dc3a70a5c 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -84,21 +84,33 @@ def normalize_trace_output(output): USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) +TERMINATE_TIMEOUT = 10 def create_process_group(*args, **kwargs): if USE_PROCESS_GROUP: kwargs['start_new_session'] = True return subprocess.Popen(*args, **kwargs) -def kill_process_group(proc): +def terminate_process_group(proc): if USE_PROCESS_GROUP: try: - os.killpg(proc.pid, signal.SIGKILL) + os.killpg(proc.pid, signal.SIGTERM) except ProcessLookupError: pass else: - proc.kill() - proc.communicate() # Clean up + proc.terminate() + + try: + proc.communicate(timeout=TERMINATE_TIMEOUT) + except subprocess.TimeoutExpired: + if USE_PROCESS_GROUP: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + proc.kill() + proc.communicate(timeout=TERMINATE_TIMEOUT) # Clean up def run_readelf(cmd): @@ -163,7 +175,7 @@ def trace(self, script_file, subcommand=None, *, timeout=None, try: stdout, _ = proc.communicate(timeout=timeout) except subprocess.TimeoutExpired: - kill_process_group(proc) + terminate_process_group(proc) raise if check_returncode and proc.returncode: raise AssertionError( @@ -361,7 +373,7 @@ def run_case(self, name, optimize_python=None): ) stdout, stderr = proc.communicate(timeout=60) except subprocess.TimeoutExpired: - kill_process_group(proc) + terminate_process_group(proc) raise AssertionError("bpftrace timed out") except (FileNotFoundError, PermissionError) as e: raise unittest.SkipTest(f"bpftrace not available: {e}") @@ -400,7 +412,7 @@ def assert_usable(self): ) stdout, stderr = proc.communicate(timeout=10) except subprocess.TimeoutExpired: - kill_process_group(proc) + terminate_process_group(proc) raise unittest.SkipTest("bpftrace timed out during usability check") except OSError as e: raise unittest.SkipTest(f"bpftrace not available: {e}") From 4532b36db5b9a43f63bf295aaf51df426e4c488d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 23 Sep 2026 13:09:32 +0300 Subject: [PATCH 4/6] gh-157672: Keep Python's signal handlers when Tk is initialized on macOS (GH-157673) Tk on macOS replaces the SIGINT, SIGHUP and SIGTERM handlers with its own, which exits the process, so Ctrl-C terminated a tkinter program instead of raising KeyboardInterrupt. Save the handlers installed by Python before Tk_Init() and restore them after it. This affects Tk 8.6.11 to 8.6.18 and 9.0 to 9.0.4. Tk itself is fixed in 8.6.19 and 9.0.5. --- Lib/test/test_tkinter/test_misc.py | 14 ++++++++++++ ...-09-17-14-00-00.gh-issue-157672.tksigs.rst | 4 ++++ Modules/_tkinter.c | 22 ++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index c9d402e11a8826..5c1eea78669859 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -3,6 +3,7 @@ import os import gc import platform +import signal import sys import textwrap import time @@ -13,6 +14,7 @@ import enum from test import support from test.support import os_helper +from test.support.isolation import runInSubprocess from test.support.script_helper import assert_python_ok from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, @@ -2149,6 +2151,18 @@ def _info_commands(widget, pattern=None): return widget.tk.splitlist(widget.tk.call('info', 'commands', pattern)) +class SignalTest(unittest.TestCase): + + @runInSubprocess() + def test_sigint_handler(self): + # gh-157672: Tk on macOS replaced the SIGINT handler with its own, + # which exits the process. + root = tkinter.Tk() + self.addCleanup(root.destroy) + with self.assertRaises(KeyboardInterrupt): + signal.raise_signal(signal.SIGINT) + + class TclObjTypeTest(AbstractTkTest, unittest.TestCase): # See FromObj() in Modules/_tkinter.c: Tcl object types are converted to # appropriate Python types. These conversions only happen in the object diff --git a/Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst b/Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst new file mode 100644 index 00000000000000..a751a2f9dc406b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-17-14-00-00.gh-issue-157672.tksigs.rst @@ -0,0 +1,4 @@ +Fix :mod:`tkinter` on macOS: creating a :class:`~tkinter.Tk` instance no +longer replaces the Python handlers of SIGINT, SIGHUP and SIGTERM with Tk's +handler which exits the process, so Ctrl-C raises :exc:`KeyboardInterrupt` +again. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index f5fb1841f0d594..746c39e6d3f4a2 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -33,6 +33,8 @@ Copyright (C) 1994 Steen Lumholt. #include "pycore_long.h" // _PyLong_IsNegative() #include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8String +#include // SIGINT + #ifdef MS_WINDOWS # include #endif @@ -227,7 +229,25 @@ Tkinter_TkInit(Tcl_Interp *interp) does not search. Mount the DLL using Zipfs if possible. */ mount_tk_dll_zip(); #endif - return Tk_Init(interp); +#ifdef __APPLE__ + /* Tk on macOS replaces the handlers of these signals with its own, + which exits the process. Keep the handlers installed by Python + (gh-157672). */ + static const int signals[] = {SIGINT, SIGHUP, SIGTERM}; + PyOS_sighandler_t handlers[Py_ARRAY_LENGTH(signals)]; + for (size_t i = 0; i < Py_ARRAY_LENGTH(signals); i++) { + handlers[i] = PyOS_getsig(signals[i]); + } +#endif + int result = Tk_Init(interp); +#ifdef __APPLE__ + for (size_t i = 0; i < Py_ARRAY_LENGTH(signals); i++) { + if (handlers[i] != SIG_DFL) { + PyOS_setsig(signals[i], handlers[i]); + } + } +#endif + return result; } /* The threading situation is complicated. Tcl is not thread-safe, except From e1be6132b20fc7a65edac8c671420fa61eabe46c Mon Sep 17 00:00:00 2001 From: stratakis Date: Wed, 23 Sep 2026 12:10:09 +0200 Subject: [PATCH 5/6] gh-156701: Increase the SystemTap usability timeout (#156720) SystemTap may compile a kernel module during its usability check (if no cached module is available). We need to allow a larger timeout for that case. --- Lib/test/test_dtrace.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index 946f6dc3a70a5c..d398c0789d24a1 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -144,6 +144,7 @@ class TraceBackend: EXTENSION = None COMMAND = None COMMAND_ARGS = [] + USABILITY_TIMEOUT = 10 def run_case(self, name, optimize_python=None): try: @@ -195,7 +196,7 @@ def trace_python(self, script_file, python_file, optimize_python=None): def assert_usable(self): try: output = self.trace(abspath("assert_usable" + self.EXTENSION), - timeout=10) + timeout=self.USABILITY_TIMEOUT) output = output.strip() except subprocess.TimeoutExpired: raise unittest.SkipTest( @@ -220,6 +221,7 @@ class SystemTapBackend(TraceBackend): EXTENSION = ".stp" COMMAND = ["stap", "-g"] PROBE_PLACEHOLDER = "@PYTHON_SYSTEMTAP_PROBE@" + USABILITY_TIMEOUT = 60 @staticmethod def quote_systemtap_string(value): From 68ed7a50226f8981788f2673677e30f1431e18a9 Mon Sep 17 00:00:00 2001 From: Locked-chess-official <13140752715@163.com> Date: Wed, 23 Sep 2026 18:54:26 +0800 Subject: [PATCH 6/6] gh-151471: Colorize the soft keyword `type` in IDLE (GH-151700) `type` is now colored as a keyword when it starts a statement and is followed by a name, as in `type Point = tuple[float, float]`. It stays a builtin elsewhere, for example in `type = type(1)`, `type(x)` or `type in (int, str)`. --- Lib/idlelib/colorizer.py | 8 +++++++- Lib/idlelib/idle_test/test_colorizer.py | 6 +++++- .../IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index ac12d8ace3575f..5f18a22d2f0507 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -42,6 +42,11 @@ def make_pat(): ]) + r"))" ) + type_softkw = ( + r"^[ \t]*" + # at beginning of line + possible indentation + r"(?Ptype)" + + r"(?=[ \t]+(?!(?:" + "|".join(keyword.kwlist) + r")\b)[^\W\d])" + ) lazy_softkw = ( # lazy new in 3.15 (+ 2 lines below). r"^[ \t]*" + # at beginning of line + possible indentation r"(?Plazy)" + @@ -59,7 +64,7 @@ def make_pat(): dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) prog = re.compile("|".join([ - builtin, comment, string, kw, + type_softkw, builtin, comment, string, kw, match_softkw, case_default, case_softkw_and_pattern, lazy_softkw, any("SYNC", [r"\n"]), @@ -75,6 +80,7 @@ def make_pat(): "CASE_SOFTKW": "KEYWORD", "CASE_DEFAULT_UNDERSCORE": "KEYWORD", "CASE_SOFTKW2": "KEYWORD", + "TYPE_SOFTKW": "KEYWORD", "LAZY_SOFTKW": "KEYWORD", } diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 8a6ab527f9387f..5bb4f5b3ff36e6 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -52,6 +52,8 @@ async def f(): await g() ''' case _:''' "match x:" + type Point = tuple[float, float] + type = type(1) """) @@ -404,6 +406,8 @@ def test_recolorize_main(self, mock_notify): ('28.25', ('STRING',)), ('28.38', ('STRING',)), ('30.0', ('STRING',)), ('31.1', ('STRING',)), + ('32.0', ('KEYWORD',)), + ('33.0', ('BUILTIN',)), ('33.1', ('BUILTIN',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -434,7 +438,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17')) eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26')) eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) - eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '33.0')) + eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '32.0')) def _assert_highlighting(self, source, tag_ranges): """Check highlighting of a given piece of code. diff --git a/Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst b/Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst new file mode 100644 index 00000000000000..cc04822f6d3321 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-17-08-53-12.gh-issue-151471.-yvE03.rst @@ -0,0 +1 @@ +Colorize the soft keyword ``type`` in IDLE.