From 71db05a12d9953a96f809d84b4d0d452a464e431 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Tue, 21 Oct 2025 08:10:01 -0400 Subject: [PATCH 1/7] gh-140406: Fix memory leak upon `__hash__` returning a non-integer (GH-140411) --- Lib/test/test_builtin.py | 10 ++++++++++ .../2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst | 2 ++ Objects/typeobject.c | 1 + 3 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 85cfe5c90f48af8..034cd5f7f9e89f3 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1184,6 +1184,16 @@ def __hash__(self): return self self.assertEqual(hash(Z(42)), hash(42)) + def test_invalid_hash_typeerror(self): + # GH-140406: The returned object from __hash__() would leak if it + # wasn't an integer. + class A: + def __hash__(self): + return 1.0 + + with self.assertRaises(TypeError): + hash(A()) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst new file mode 100644 index 000000000000000..3506ba42581faaa --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst @@ -0,0 +1,2 @@ +Fix memory leak when an object's :meth:`~object.__hash__` method returns an +object that isn't an :class:`int`. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 29233c1959c4d07..721d48712446a0f 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -10569,6 +10569,7 @@ slot_tp_hash(PyObject *self) return PyObject_HashNotImplemented(self); } if (!PyLong_Check(res)) { + Py_DECREF(res); PyErr_SetString(PyExc_TypeError, "__hash__ method should return an integer"); return -1; From 4adf08ae463e8b5311c5a7662ab26f8d0d14381d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 21 Oct 2025 13:37:06 +0100 Subject: [PATCH 2/7] GH-139436: Remove references to downloadable PDF documentation (#140416) --- Doc/faq/general.rst | 2 +- README.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 578777d7f236211..698f2117ff5c649 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -186,7 +186,7 @@ How do I get documentation on Python? ------------------------------------- The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are +at https://docs.python.org/3/. EPUB, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx diff --git a/README.rst b/README.rst index c6f4549f21b11b1..a228aafb09c7049 100644 --- a/README.rst +++ b/README.rst @@ -153,7 +153,7 @@ Documentation updated daily. It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is downloadable in HTML, EPUB, and reStructuredText formats; the latter version is primarily for documentation authors, translators, and people with special formatting requirements. @@ -232,4 +232,4 @@ This Python distribution contains *no* GNU General Public License (GPL) code, so it may be used in proprietary projects. There are interfaces to some GNU code but these are entirely optional. -All trademarks referenced herein are property of their respective holders. \ No newline at end of file +All trademarks referenced herein are property of their respective holders. From 0c01090ad957de4625f504ce4f29df0a05d09fba Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 21 Oct 2025 15:22:15 +0100 Subject: [PATCH 3/7] GH-139951: Fix major GC performance regression (GH-140262) * Count number of actually tracked objects, instead of trackable objects. This ensures that untracking tuples has the desired effect of reducing GC overhead * Do not track most untrackable tuples during creation. This prevents large numbers of small tuples causing execessive GCs. --- Include/internal/pycore_gc.h | 23 ++++++++-- Lib/test/test_gc.py | 17 ++++---- ...-10-17-18-03-12.gh-issue-139951.IdwM2O.rst | 7 +++ Objects/tupleobject.c | 39 +++++++++++++++-- Python/gc.c | 43 ++++++++++--------- 5 files changed, 93 insertions(+), 36 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index a6519aa086309de..fd284d0e4ecc2f5 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -205,6 +205,12 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #endif } +extern void _Py_ScheduleGC(PyThreadState *tstate); + +#ifndef Py_GIL_DISABLED +extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); +#endif + /* Tell the GC to track this object. * @@ -238,14 +244,19 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = &interp->gc.young.head; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + PyGC_Head *generation0 = &gcstate->young.head; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ interp->gc.visited_space; + uintptr_t not_visited = 1 ^ gcstate->visited_space; gc->_gc_next = ((uintptr_t)generation0) | not_visited; generation0->_gc_prev = (uintptr_t)gc; + gcstate->young.count++; /* number of tracked GC objects */ + gcstate->heap_size++; + if (gcstate->young.count > gcstate->young.threshold) { + _Py_TriggerGC(gcstate); + } #endif } @@ -280,6 +291,11 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + if (gcstate->young.count > 0) { + gcstate->young.count--; + } + gcstate->heap_size--; #endif } @@ -343,7 +359,6 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 08055bb8e6bd41c..10c3a6221077143 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1473,10 +1473,11 @@ def callback(ignored): # The free-threaded build doesn't have multiple generations, so # just trigger a GC manually. gc.collect() + assert not detector.gc_happened while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 100000: + self.fail("gc didn't happen after 100000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1548,8 +1549,8 @@ def __del__(self): gc.collect() while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 50000: + self.fail("gc didn't happen after 50000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1566,8 +1567,8 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 100000: + self.fail("gc didn't happen after 100000 iterations") junk.append([]) # this will eventually trigger gc try: @@ -1577,11 +1578,11 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 10000: + if i > 100000: break junk.append([]) # this may eventually trigger gc (if it is enabled) - self.assertEqual(i, 10001) + self.assertEqual(i, 100001) finally: gc.enable() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst new file mode 100644 index 000000000000000..e03996188a7e227 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst @@ -0,0 +1,7 @@ +Fixes a regression in GC performance for a growing heap composed mostly of +small tuples. + +* Counts number of actually tracked objects, instead of trackable objects. + This ensures that untracking tuples has the desired effect of reducing GC overhead. +* Does not track most untrackable tuples during creation. + This prevents large numbers of small tuples causing excessive GCs. diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index cd90b06d499faf7..169ac69701da110 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -156,6 +156,18 @@ _PyTuple_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +/* Fast, but conservative check if an object maybe tracked + May return true for an object that is not tracked, + Will always return true for an object that is tracked. + This is a temporary workaround until _PyObject_GC_IS_TRACKED + becomes fast and safe to call on non-GC objects. +*/ +static bool +maybe_tracked(PyObject *ob) +{ + return _PyType_IS_GC(Py_TYPE(ob)); +} + PyObject * PyTuple_Pack(Py_ssize_t n, ...) { @@ -163,6 +175,7 @@ PyTuple_Pack(Py_ssize_t n, ...) PyObject *o; PyObject **items; va_list vargs; + bool track = false; if (n == 0) { return tuple_get_empty(); @@ -177,10 +190,15 @@ PyTuple_Pack(Py_ssize_t n, ...) items = result->ob_item; for (i = 0; i < n; i++) { o = va_arg(vargs, PyObject *); + if (!track && maybe_tracked(o)) { + track = true; + } items[i] = Py_NewRef(o); } va_end(vargs); - _PyObject_GC_TRACK(result); + if (track) { + _PyObject_GC_TRACK(result); + } return (PyObject *)result; } @@ -377,11 +395,17 @@ PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = src[i]; + if (!track && maybe_tracked(item)) { + track = true; + } dst[i] = Py_NewRef(item); } - _PyObject_GC_TRACK(tuple); + if (track) { + _PyObject_GC_TRACK(tuple); + } return (PyObject *)tuple; } @@ -396,10 +420,17 @@ _PyTuple_FromStackRefStealOnSuccess(const _PyStackRef *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { - dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + PyObject *item = PyStackRef_AsPyObjectSteal(src[i]); + if (!track && maybe_tracked(item)) { + track = true; + } + dst[i] = item; + } + if (track) { + _PyObject_GC_TRACK(tuple); } - _PyObject_GC_TRACK(tuple); return (PyObject *)tuple; } diff --git a/Python/gc.c b/Python/gc.c index fd724301c769bf8..a1f3d86d91036b5 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1639,7 +1639,7 @@ assess_work_to_do(GCState *gcstate) scale_factor = 2; } intptr_t new_objects = gcstate->young.count; - intptr_t max_heap_fraction = new_objects*3/2; + intptr_t max_heap_fraction = new_objects*2; intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; if (heap_fraction > max_heap_fraction) { heap_fraction = max_heap_fraction; @@ -1654,6 +1654,9 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) GC_STAT_ADD(1, collections, 1); GCState *gcstate = &tstate->interp->gc; gcstate->work_to_do += assess_work_to_do(gcstate); + if (gcstate->work_to_do < 0) { + return; + } untrack_tuples(&gcstate->young.head); if (gcstate->phase == GC_PHASE_MARK) { Py_ssize_t objects_marked = mark_at_start(tstate); @@ -1696,7 +1699,6 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) gc_collect_region(tstate, &increment, &survivors, stats); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; add_stats(gcstate, 1, stats); @@ -2299,21 +2301,11 @@ _Py_ScheduleGC(PyThreadState *tstate) } void -_PyObject_GC_Link(PyObject *op) +_Py_TriggerGC(struct _gc_runtime_state *gcstate) { - PyGC_Head *gc = AS_GC(op); - // gc must be correctly aligned - _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - gc->_gc_next = 0; - gc->_gc_prev = 0; - gcstate->young.count++; /* number of allocated GC objects */ - gcstate->heap_size++; - if (gcstate->young.count > gcstate->young.threshold && - gcstate->enabled && - gcstate->young.threshold && + if (gcstate->enabled && + gcstate->young.threshold != 0 && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -2321,6 +2313,17 @@ _PyObject_GC_Link(PyObject *op) } } +void +_PyObject_GC_Link(PyObject *op) +{ + PyGC_Head *gc = AS_GC(op); + // gc must be correctly aligned + _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); + gc->_gc_next = 0; + gc->_gc_prev = 0; + +} + void _Py_RunGC(PyThreadState *tstate) { @@ -2427,6 +2430,11 @@ PyObject_GC_Del(void *op) PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); + GCState *gcstate = get_gc_state(); + if (gcstate->young.count > 0) { + gcstate->young.count--; + } + gcstate->heap_size--; #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -2440,11 +2448,6 @@ PyObject_GC_Del(void *op) PyErr_SetRaisedException(exc); #endif } - GCState *gcstate = get_gc_state(); - if (gcstate->young.count > 0) { - gcstate->young.count--; - } - gcstate->heap_size--; PyObject_Free(((char *)op)-presize); } From c788bfb80e0e8b312d31db810de4c090a1059a71 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Tue, 21 Oct 2025 08:54:00 -0700 Subject: [PATCH 4/7] gh-140239: Check for statx on Android (#140395) Android has Linux's statx, but MACHDEP is "android" on Android, so configure doesn't check for statx on Android. Base the check for statx on ac_sys_system instead, which is "Linux-android" on Android, "Linux" on other Linux distributions, and "AIX" on AIX (which has an incompatible function named statx). --- .../2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst | 2 +- configure | 13 +++++++++---- configure.ac | 9 +++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst b/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst index f196ab0964d2c50..713f022c9949583 100644 --- a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst +++ b/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst @@ -1 +1 @@ -Check ``statx`` availability only in Linux platforms +Check ``statx`` availability only on Linux (including Android). diff --git a/configure b/configure index 95e231f406a9a3e..f95355e2f57d69b 100755 --- a/configure +++ b/configure @@ -20392,16 +20392,21 @@ then : fi -# Check statx availability in Linux -if test "$MACHDEP" = "linux"; then - ac_fn_c_check_func "$LINENO" "statx" "ac_cv_func_statx" +# os.statx uses Linux's statx function. AIX also has a function named statx, +# but it's unrelated. Check only on Linux (including Android). +case $ac_sys_system in #( + Linux*) : + ac_fn_c_check_func "$LINENO" "statx" "ac_cv_func_statx" if test "x$ac_cv_func_statx" = xyes then : printf "%s\n" "#define HAVE_STATX 1" >>confdefs.h fi -fi + ;; #( + *) : + ;; +esac # Force lchmod off for Linux. Linux disallows changing the mode of symbolic # links. Some libc implementations have a stub lchmod implementation that always diff --git a/configure.ac b/configure.ac index 65c375796337543..4d0e5a1ca1d0036 100644 --- a/configure.ac +++ b/configure.ac @@ -5258,10 +5258,11 @@ AC_CHECK_FUNCS([ \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) -# Check statx availability in Linux -if test "$MACHDEP" = "linux"; then - AC_CHECK_FUNCS([statx]) -fi +# os.statx uses Linux's statx function. AIX also has a function named statx, +# but it's unrelated. Check only on Linux (including Android). +AS_CASE([$ac_sys_system], + [Linux*], [AC_CHECK_FUNCS([statx])] +) # Force lchmod off for Linux. Linux disallows changing the mode of symbolic # links. Some libc implementations have a stub lchmod implementation that always From 95c257e2e691456140e79bd98d1674cbd289eb38 Mon Sep 17 00:00:00 2001 From: David Ellis Date: Tue, 21 Oct 2025 16:57:43 +0100 Subject: [PATCH 5/7] gh-138764: annotationlib: Make `call_annotate_function` fallback to using `VALUE` annotations if both the requested format and `VALUE_WITH_FAKE_GLOBALS` are not implemented (#138803) --- Doc/library/annotationlib.rst | 15 ++ Lib/annotationlib.py | 15 ++ Lib/test/test_annotationlib.py | 176 ++++++++++++++++++ ...-09-12-09-34-37.gh-issue-138764.mokHoY.rst | 3 + 4 files changed, 209 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index d6f5055955e8cf2..40f2a6dc30460b7 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -340,14 +340,29 @@ Functions * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, the :attr:`!object.__annotate__` function is called if it exists. + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it does not exist either, :attr:`!object.__annotations__` is tried again and any error from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + * STRING: If :attr:`!object.__annotate__` exists, it is called first; otherwise, :attr:`!object.__annotations__` is used and stringified using :func:`annotations_to_string`. + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two different but equivalent dicts. diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 544e069626d0d96..81886a0467d0018 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -695,6 +695,18 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) closure = _build_closure( @@ -753,6 +765,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) except Exception: pass else: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 2c5bf2b3417344f..8da4ff096e7593b 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1194,6 +1194,25 @@ class RaisesAttributeError: }, ) + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): @@ -1214,6 +1233,163 @@ def evaluate(format, exc=NotImplementedError): ) +class TestCallAnnotateFunction(unittest.TestCase): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": float}) + + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": int}) + + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "float"}) + + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "str"}) + + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) + + class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): class Meta(type): diff --git a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst new file mode 100644 index 000000000000000..85ebef8ff11d5ce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst @@ -0,0 +1,3 @@ +Prevent :func:`annotationlib.call_annotate_function` from calling ``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in a fake globals namespace with empty globals. + +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` annotations in the case that neither their own format, nor ``VALUE_WITH_FAKE_GLOBALS`` are supported. From b4d230736709235f90fe635f3322973cc9fadd21 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 21 Oct 2025 17:02:43 +0100 Subject: [PATCH 6/7] Use ``sphinxext-opengraph`` v0.13.0 (GH-140425) --- Doc/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 7b7286429a10414..d5f7b473c3aa844 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -11,7 +11,7 @@ sphinx~=8.2.0 blurb -sphinxext-opengraph~=0.12.0 +sphinxext-opengraph~=0.13.0 sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need From 60df1d7e0c8eef4bb7b9c41e2b65957ab4a4cd34 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Tue, 21 Oct 2025 09:13:14 -0700 Subject: [PATCH 7/7] gh-83714: Check for struct statx members in configure (#140402) Some systems have the definitions of the mask bits without having the corresponding members in struct statx. Add configure checks for members added after Linux 4.11 (when statx itself was added). --- Doc/library/os.rst | 6 ++--- Modules/posixmodule.c | 20 ++++++++--------- configure | 51 +++++++++++++++++++++++++++++++++++++++++++ configure.ac | 11 ++++++++++ pyconfig.h.in | 17 +++++++++++++++ 5 files changed, 92 insertions(+), 13 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index ef08e1bfa51a6f5..50f6886c89df9e1 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3566,9 +3566,9 @@ features: STATX_WRITE_ATOMIC STATX_DIO_READ_ALIGN - Bitflags for use in the *mask* parameter to :func:`os.statx`. Flags - including and after :const:`!STATX_MNT_ID` are only available when their - corresponding members in :class:`statx_result` are available. + Bitflags for use in the *mask* parameter to :func:`os.statx`. Some of these + flags may be available even when their corresponding members in + :class:`statx_result` are not available. .. availability:: Linux >= 4.11 with glibc >= 2.28. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 1cfb068680cc4b7..d0b9eed8c8208b7 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3364,15 +3364,15 @@ static PyMemberDef pystatx_result_members[] = { STATX_GET_UINT(st_uid, stx_uid, STATX_UID) STATX_GET_UINT(st_gid, stx_gid, STATX_GID) STATX_GET_UINT(st_nlink, stx_nlink, STATX_NLINK) -#ifdef STATX_DIOALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN STATX_GET_UINT(stx_dio_mem_align, stx_dio_mem_align, STATX_DIOALIGN) STATX_GET_UINT(stx_dio_offset_align, stx_dio_offset_align, STATX_DIOALIGN) #endif -#ifdef STATX_DIO_READ_ALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN STATX_GET_UINT(stx_dio_read_offset_align, stx_dio_read_offset_align, STATX_DIO_READ_ALIGN) #endif -#ifdef STATX_WRITE_ATOMIC +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN STATX_GET_UINT(stx_atomic_write_unit_min, stx_atomic_write_unit_min, STATX_WRITE_ATOMIC) STATX_GET_UINT(stx_atomic_write_unit_max, stx_atomic_write_unit_max, @@ -3401,10 +3401,10 @@ STATX_GET_UINT(stx_atomic_write_unit_max_opt, stx_atomic_write_unit_max_opt, STATX_GET_ULONGLONG(st_blocks, stx_blocks, STATX_BLOCKS) STATX_GET_ULONGLONG(st_ino, stx_ino, STATX_INO) STATX_GET_ULONGLONG(st_size, stx_size, STATX_SIZE) -#ifdef STATX_MNT_ID +#ifdef HAVE_STRUCT_STATX_STX_MNT_ID STATX_GET_ULONGLONG(stx_mnt_id, stx_mnt_id, STATX_MNT_ID) #endif -#ifdef STATX_SUBVOL +#ifdef HAVE_STRUCT_STATX_STX_SUBVOL STATX_GET_ULONGLONG(stx_subvol, stx_subvol, STATX_SUBVOL) #endif @@ -3463,17 +3463,17 @@ static PyGetSetDef pystatx_result_getset[] = { G(st_ctime_ns, "time of last change in nanoseconds"), G(st_mtime, "time of last modification"), G(st_mtime_ns, "time of last modification in nanoseconds"), -#ifdef STATX_MNT_ID +#ifdef HAVE_STRUCT_STATX_STX_MNT_ID G(stx_mnt_id, "mount ID"), #endif -#ifdef STATX_DIOALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN G(stx_dio_mem_align, "direct I/O memory buffer alignment"), G(stx_dio_offset_align, "direct I/O file offset alignment"), #endif -#ifdef STATX_SUBVOL +#ifdef HAVE_STRUCT_STATX_STX_SUBVOL G(stx_subvol, "subvolume ID"), #endif -#ifdef STATX_WRITE_ATOMIC +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN G(stx_atomic_write_unit_min, "minimum size for direct I/O with torn-write protection"), G(stx_atomic_write_unit_max, @@ -3481,7 +3481,7 @@ static PyGetSetDef pystatx_result_getset[] = { G(stx_atomic_write_segments_max, "maximum iovecs for direct I/O with torn-write protection"), #endif -#ifdef STATX_DIO_READ_ALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN G(stx_dio_read_offset_align, "direct I/O file offset alignment for reads"), #endif #ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT diff --git a/configure b/configure index f95355e2f57d69b..31eb0baa9242078 100755 --- a/configure +++ b/configure @@ -25144,6 +25144,57 @@ fi if test "$ac_cv_func_statx" = yes; then + # Some systems have the definitions of the mask bits without having the + # corresponding members in struct statx. Check for members added after Linux + # 4.11 (when statx itself was added). + ac_fn_c_check_member "$LINENO" "struct statx" "stx_mnt_id" "ac_cv_member_struct_statx_stx_mnt_id" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_mnt_id" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_MNT_ID 1" >>confdefs.h + + +fi + + ac_fn_c_check_member "$LINENO" "struct statx" "stx_dio_mem_align" "ac_cv_member_struct_statx_stx_dio_mem_align" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_dio_mem_align" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN 1" >>confdefs.h + + +fi + + # stx_dio_offset_align was added together with stx_dio_mem_align + ac_fn_c_check_member "$LINENO" "struct statx" "stx_subvol" "ac_cv_member_struct_statx_stx_subvol" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_subvol" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_SUBVOL 1" >>confdefs.h + + +fi + + ac_fn_c_check_member "$LINENO" "struct statx" "stx_atomic_write_unit_min" "ac_cv_member_struct_statx_stx_atomic_write_unit_min" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_atomic_write_unit_min" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN 1" >>confdefs.h + + +fi + + # stx_atomic_write_unit_max and stx_atomic_write_segments_max were added + # together with stx_atomic_write_unit_min + ac_fn_c_check_member "$LINENO" "struct statx" "stx_dio_read_offset_align" "ac_cv_member_struct_statx_stx_dio_read_offset_align" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_dio_read_offset_align" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN 1" >>confdefs.h + + +fi + # stx_atomic_write_unit_max_opt was added in Linux 6.16, but is controlled by # the STATX_WRITE_ATOMIC mask bit added in Linux 6.11, so having the mask bit # doesn't imply having the member. diff --git a/configure.ac b/configure.ac index 4d0e5a1ca1d0036..6f29362f46735b5 100644 --- a/configure.ac +++ b/configure.ac @@ -5827,6 +5827,17 @@ AC_CHECK_MEMBERS([struct passwd.pw_gecos, struct passwd.pw_passwd], [], [], [[ AC_CHECK_MEMBERS([siginfo_t.si_band], [], [], [[@%:@include ]]) if test "$ac_cv_func_statx" = yes; then + # Some systems have the definitions of the mask bits without having the + # corresponding members in struct statx. Check for members added after Linux + # 4.11 (when statx itself was added). + AC_CHECK_MEMBERS([struct statx.stx_mnt_id]) + AC_CHECK_MEMBERS([struct statx.stx_dio_mem_align]) + # stx_dio_offset_align was added together with stx_dio_mem_align + AC_CHECK_MEMBERS([struct statx.stx_subvol]) + AC_CHECK_MEMBERS([struct statx.stx_atomic_write_unit_min]) + # stx_atomic_write_unit_max and stx_atomic_write_segments_max were added + # together with stx_atomic_write_unit_min + AC_CHECK_MEMBERS([struct statx.stx_dio_read_offset_align]) # stx_atomic_write_unit_max_opt was added in Linux 6.16, but is controlled by # the STATX_WRITE_ATOMIC mask bit added in Linux 6.11, so having the mask bit # doesn't imply having the member. diff --git a/pyconfig.h.in b/pyconfig.h.in index 092f96e7152e117..fb12079bafa95e0 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1334,6 +1334,23 @@ statx'. */ #undef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT +/* Define to 1 if 'stx_atomic_write_unit_min' is a member of 'struct statx'. + */ +#undef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN + +/* Define to 1 if 'stx_dio_mem_align' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN + +/* Define to 1 if 'stx_dio_read_offset_align' is a member of 'struct statx'. + */ +#undef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN + +/* Define to 1 if 'stx_mnt_id' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_MNT_ID + +/* Define to 1 if 'stx_subvol' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_SUBVOL + /* Define to 1 if 'st_birthtime' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BIRTHTIME