diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 578777d7f23621..698f2117ff5c64 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -186,7 +186,7 @@ How do I get documentation on Python? ------------------------------------- The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are +at https://docs.python.org/3/. EPUB, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index d6f5055955e8cf..40f2a6dc30460b 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -340,14 +340,29 @@ Functions * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, the :attr:`!object.__annotate__` function is called if it exists. + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it does not exist either, :attr:`!object.__annotations__` is tried again and any error from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + * STRING: If :attr:`!object.__annotate__` exists, it is called first; otherwise, :attr:`!object.__annotations__` is used and stringified using :func:`annotations_to_string`. + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two different but equivalent dicts. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index ef08e1bfa51a6f..50f6886c89df9e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3566,9 +3566,9 @@ features: STATX_WRITE_ATOMIC STATX_DIO_READ_ALIGN - Bitflags for use in the *mask* parameter to :func:`os.statx`. Flags - including and after :const:`!STATX_MNT_ID` are only available when their - corresponding members in :class:`statx_result` are available. + Bitflags for use in the *mask* parameter to :func:`os.statx`. Some of these + flags may be available even when their corresponding members in + :class:`statx_result` are not available. .. availability:: Linux >= 4.11 with glibc >= 2.28. diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 7b7286429a1041..d5f7b473c3aa84 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -11,7 +11,7 @@ sphinx~=8.2.0 blurb -sphinxext-opengraph~=0.12.0 +sphinxext-opengraph~=0.13.0 sphinx-notfound-page~=1.0.0 # The theme used by the documentation is stored separately, so we need diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index a6519aa086309d..fd284d0e4ecc2f 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -205,6 +205,12 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #endif } +extern void _Py_ScheduleGC(PyThreadState *tstate); + +#ifndef Py_GIL_DISABLED +extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); +#endif + /* Tell the GC to track this object. * @@ -238,14 +244,19 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = &interp->gc.young.head; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + PyGC_Head *generation0 = &gcstate->young.head; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ interp->gc.visited_space; + uintptr_t not_visited = 1 ^ gcstate->visited_space; gc->_gc_next = ((uintptr_t)generation0) | not_visited; generation0->_gc_prev = (uintptr_t)gc; + gcstate->young.count++; /* number of tracked GC objects */ + gcstate->heap_size++; + if (gcstate->young.count > gcstate->young.threshold) { + _Py_TriggerGC(gcstate); + } #endif } @@ -280,6 +291,11 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + if (gcstate->young.count > 0) { + gcstate->young.count--; + } + gcstate->heap_size--; #endif } @@ -343,7 +359,6 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 544e069626d0d9..81886a0467d001 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -695,6 +695,18 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) closure = _build_closure( @@ -753,6 +765,9 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) except Exception: pass else: diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 2c5bf2b3417344..8da4ff096e7593 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -1194,6 +1194,25 @@ class RaisesAttributeError: }, ) + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): @@ -1214,6 +1233,163 @@ def evaluate(format, exc=NotImplementedError): ) +class TestCallAnnotateFunction(unittest.TestCase): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": float}) + + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": int}) + + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "float"}) + + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "str"}) + + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) + + class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): class Meta(type): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 85cfe5c90f48af..034cd5f7f9e89f 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1184,6 +1184,16 @@ def __hash__(self): return self self.assertEqual(hash(Z(42)), hash(42)) + def test_invalid_hash_typeerror(self): + # GH-140406: The returned object from __hash__() would leak if it + # wasn't an integer. + class A: + def __hash__(self): + return 1.0 + + with self.assertRaises(TypeError): + hash(A()) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 08055bb8e6bd41..10c3a622107714 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1473,10 +1473,11 @@ def callback(ignored): # The free-threaded build doesn't have multiple generations, so # just trigger a GC manually. gc.collect() + assert not detector.gc_happened while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 100000: + self.fail("gc didn't happen after 100000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1548,8 +1549,8 @@ def __del__(self): gc.collect() while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 50000: + self.fail("gc didn't happen after 50000 iterations") self.assertEqual(len(ouch), 0) junk.append([]) # this will eventually trigger gc @@ -1566,8 +1567,8 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 10000: - self.fail("gc didn't happen after 10000 iterations") + if i > 100000: + self.fail("gc didn't happen after 100000 iterations") junk.append([]) # this will eventually trigger gc try: @@ -1577,11 +1578,11 @@ def test_indirect_calls_with_gc_disabled(self): detector = GC_Detector() while not detector.gc_happened: i += 1 - if i > 10000: + if i > 100000: break junk.append([]) # this may eventually trigger gc (if it is enabled) - self.assertEqual(i, 10001) + self.assertEqual(i, 100001) finally: gc.enable() diff --git a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst b/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst index f196ab0964d2c5..713f022c994958 100644 --- a/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst +++ b/Misc/NEWS.d/next/Build/2025-10-17-11-33-45.gh-issue-140239._k-GgW.rst @@ -1 +1 @@ -Check ``statx`` availability only in Linux platforms +Check ``statx`` availability only on Linux (including Android). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst new file mode 100644 index 00000000000000..e03996188a7e22 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-17-18-03-12.gh-issue-139951.IdwM2O.rst @@ -0,0 +1,7 @@ +Fixes a regression in GC performance for a growing heap composed mostly of +small tuples. + +* Counts number of actually tracked objects, instead of trackable objects. + This ensures that untracking tuples has the desired effect of reducing GC overhead. +* Does not track most untrackable tuples during creation. + This prevents large numbers of small tuples causing excessive GCs. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst new file mode 100644 index 00000000000000..3506ba42581faa --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-21-06-51-50.gh-issue-140406.0gJs8M.rst @@ -0,0 +1,2 @@ +Fix memory leak when an object's :meth:`~object.__hash__` method returns an +object that isn't an :class:`int`. diff --git a/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst new file mode 100644 index 00000000000000..85ebef8ff11d5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-12-09-34-37.gh-issue-138764.mokHoY.rst @@ -0,0 +1,3 @@ +Prevent :func:`annotationlib.call_annotate_function` from calling ``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in a fake globals namespace with empty globals. + +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` annotations in the case that neither their own format, nor ``VALUE_WITH_FAKE_GLOBALS`` are supported. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 1cfb068680cc4b..d0b9eed8c8208b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3364,15 +3364,15 @@ static PyMemberDef pystatx_result_members[] = { STATX_GET_UINT(st_uid, stx_uid, STATX_UID) STATX_GET_UINT(st_gid, stx_gid, STATX_GID) STATX_GET_UINT(st_nlink, stx_nlink, STATX_NLINK) -#ifdef STATX_DIOALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN STATX_GET_UINT(stx_dio_mem_align, stx_dio_mem_align, STATX_DIOALIGN) STATX_GET_UINT(stx_dio_offset_align, stx_dio_offset_align, STATX_DIOALIGN) #endif -#ifdef STATX_DIO_READ_ALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN STATX_GET_UINT(stx_dio_read_offset_align, stx_dio_read_offset_align, STATX_DIO_READ_ALIGN) #endif -#ifdef STATX_WRITE_ATOMIC +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN STATX_GET_UINT(stx_atomic_write_unit_min, stx_atomic_write_unit_min, STATX_WRITE_ATOMIC) STATX_GET_UINT(stx_atomic_write_unit_max, stx_atomic_write_unit_max, @@ -3401,10 +3401,10 @@ STATX_GET_UINT(stx_atomic_write_unit_max_opt, stx_atomic_write_unit_max_opt, STATX_GET_ULONGLONG(st_blocks, stx_blocks, STATX_BLOCKS) STATX_GET_ULONGLONG(st_ino, stx_ino, STATX_INO) STATX_GET_ULONGLONG(st_size, stx_size, STATX_SIZE) -#ifdef STATX_MNT_ID +#ifdef HAVE_STRUCT_STATX_STX_MNT_ID STATX_GET_ULONGLONG(stx_mnt_id, stx_mnt_id, STATX_MNT_ID) #endif -#ifdef STATX_SUBVOL +#ifdef HAVE_STRUCT_STATX_STX_SUBVOL STATX_GET_ULONGLONG(stx_subvol, stx_subvol, STATX_SUBVOL) #endif @@ -3463,17 +3463,17 @@ static PyGetSetDef pystatx_result_getset[] = { G(st_ctime_ns, "time of last change in nanoseconds"), G(st_mtime, "time of last modification"), G(st_mtime_ns, "time of last modification in nanoseconds"), -#ifdef STATX_MNT_ID +#ifdef HAVE_STRUCT_STATX_STX_MNT_ID G(stx_mnt_id, "mount ID"), #endif -#ifdef STATX_DIOALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN G(stx_dio_mem_align, "direct I/O memory buffer alignment"), G(stx_dio_offset_align, "direct I/O file offset alignment"), #endif -#ifdef STATX_SUBVOL +#ifdef HAVE_STRUCT_STATX_STX_SUBVOL G(stx_subvol, "subvolume ID"), #endif -#ifdef STATX_WRITE_ATOMIC +#ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN G(stx_atomic_write_unit_min, "minimum size for direct I/O with torn-write protection"), G(stx_atomic_write_unit_max, @@ -3481,7 +3481,7 @@ static PyGetSetDef pystatx_result_getset[] = { G(stx_atomic_write_segments_max, "maximum iovecs for direct I/O with torn-write protection"), #endif -#ifdef STATX_DIO_READ_ALIGN +#ifdef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN G(stx_dio_read_offset_align, "direct I/O file offset alignment for reads"), #endif #ifdef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index cd90b06d499faf..169ac69701da11 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -156,6 +156,18 @@ _PyTuple_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +/* Fast, but conservative check if an object maybe tracked + May return true for an object that is not tracked, + Will always return true for an object that is tracked. + This is a temporary workaround until _PyObject_GC_IS_TRACKED + becomes fast and safe to call on non-GC objects. +*/ +static bool +maybe_tracked(PyObject *ob) +{ + return _PyType_IS_GC(Py_TYPE(ob)); +} + PyObject * PyTuple_Pack(Py_ssize_t n, ...) { @@ -163,6 +175,7 @@ PyTuple_Pack(Py_ssize_t n, ...) PyObject *o; PyObject **items; va_list vargs; + bool track = false; if (n == 0) { return tuple_get_empty(); @@ -177,10 +190,15 @@ PyTuple_Pack(Py_ssize_t n, ...) items = result->ob_item; for (i = 0; i < n; i++) { o = va_arg(vargs, PyObject *); + if (!track && maybe_tracked(o)) { + track = true; + } items[i] = Py_NewRef(o); } va_end(vargs); - _PyObject_GC_TRACK(result); + if (track) { + _PyObject_GC_TRACK(result); + } return (PyObject *)result; } @@ -377,11 +395,17 @@ PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = src[i]; + if (!track && maybe_tracked(item)) { + track = true; + } dst[i] = Py_NewRef(item); } - _PyObject_GC_TRACK(tuple); + if (track) { + _PyObject_GC_TRACK(tuple); + } return (PyObject *)tuple; } @@ -396,10 +420,17 @@ _PyTuple_FromStackRefStealOnSuccess(const _PyStackRef *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { - dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + PyObject *item = PyStackRef_AsPyObjectSteal(src[i]); + if (!track && maybe_tracked(item)) { + track = true; + } + dst[i] = item; + } + if (track) { + _PyObject_GC_TRACK(tuple); } - _PyObject_GC_TRACK(tuple); return (PyObject *)tuple; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 29233c1959c4d0..721d48712446a0 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -10569,6 +10569,7 @@ slot_tp_hash(PyObject *self) return PyObject_HashNotImplemented(self); } if (!PyLong_Check(res)) { + Py_DECREF(res); PyErr_SetString(PyExc_TypeError, "__hash__ method should return an integer"); return -1; diff --git a/Python/gc.c b/Python/gc.c index fd724301c769bf..a1f3d86d91036b 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1639,7 +1639,7 @@ assess_work_to_do(GCState *gcstate) scale_factor = 2; } intptr_t new_objects = gcstate->young.count; - intptr_t max_heap_fraction = new_objects*3/2; + intptr_t max_heap_fraction = new_objects*2; intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; if (heap_fraction > max_heap_fraction) { heap_fraction = max_heap_fraction; @@ -1654,6 +1654,9 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) GC_STAT_ADD(1, collections, 1); GCState *gcstate = &tstate->interp->gc; gcstate->work_to_do += assess_work_to_do(gcstate); + if (gcstate->work_to_do < 0) { + return; + } untrack_tuples(&gcstate->young.head); if (gcstate->phase == GC_PHASE_MARK) { Py_ssize_t objects_marked = mark_at_start(tstate); @@ -1696,7 +1699,6 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) gc_collect_region(tstate, &increment, &survivors, stats); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; add_stats(gcstate, 1, stats); @@ -2299,21 +2301,11 @@ _Py_ScheduleGC(PyThreadState *tstate) } void -_PyObject_GC_Link(PyObject *op) +_Py_TriggerGC(struct _gc_runtime_state *gcstate) { - PyGC_Head *gc = AS_GC(op); - // gc must be correctly aligned - _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); - PyThreadState *tstate = _PyThreadState_GET(); - GCState *gcstate = &tstate->interp->gc; - gc->_gc_next = 0; - gc->_gc_prev = 0; - gcstate->young.count++; /* number of allocated GC objects */ - gcstate->heap_size++; - if (gcstate->young.count > gcstate->young.threshold && - gcstate->enabled && - gcstate->young.threshold && + if (gcstate->enabled && + gcstate->young.threshold != 0 && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -2321,6 +2313,17 @@ _PyObject_GC_Link(PyObject *op) } } +void +_PyObject_GC_Link(PyObject *op) +{ + PyGC_Head *gc = AS_GC(op); + // gc must be correctly aligned + _PyObject_ASSERT(op, ((uintptr_t)gc & (sizeof(uintptr_t)-1)) == 0); + gc->_gc_next = 0; + gc->_gc_prev = 0; + +} + void _Py_RunGC(PyThreadState *tstate) { @@ -2427,6 +2430,11 @@ PyObject_GC_Del(void *op) PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); + GCState *gcstate = get_gc_state(); + if (gcstate->young.count > 0) { + gcstate->young.count--; + } + gcstate->heap_size--; #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -2440,11 +2448,6 @@ PyObject_GC_Del(void *op) PyErr_SetRaisedException(exc); #endif } - GCState *gcstate = get_gc_state(); - if (gcstate->young.count > 0) { - gcstate->young.count--; - } - gcstate->heap_size--; PyObject_Free(((char *)op)-presize); } diff --git a/README.rst b/README.rst index c6f4549f21b11b..a228aafb09c704 100644 --- a/README.rst +++ b/README.rst @@ -153,7 +153,7 @@ Documentation updated daily. It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is downloadable in HTML, EPUB, and reStructuredText formats; the latter version is primarily for documentation authors, translators, and people with special formatting requirements. @@ -232,4 +232,4 @@ This Python distribution contains *no* GNU General Public License (GPL) code, so it may be used in proprietary projects. There are interfaces to some GNU code but these are entirely optional. -All trademarks referenced herein are property of their respective holders. \ No newline at end of file +All trademarks referenced herein are property of their respective holders. diff --git a/configure b/configure index 95e231f406a9a3..31eb0baa924207 100755 --- a/configure +++ b/configure @@ -20392,16 +20392,21 @@ then : fi -# Check statx availability in Linux -if test "$MACHDEP" = "linux"; then - ac_fn_c_check_func "$LINENO" "statx" "ac_cv_func_statx" +# os.statx uses Linux's statx function. AIX also has a function named statx, +# but it's unrelated. Check only on Linux (including Android). +case $ac_sys_system in #( + Linux*) : + ac_fn_c_check_func "$LINENO" "statx" "ac_cv_func_statx" if test "x$ac_cv_func_statx" = xyes then : printf "%s\n" "#define HAVE_STATX 1" >>confdefs.h fi -fi + ;; #( + *) : + ;; +esac # Force lchmod off for Linux. Linux disallows changing the mode of symbolic # links. Some libc implementations have a stub lchmod implementation that always @@ -25139,6 +25144,57 @@ fi if test "$ac_cv_func_statx" = yes; then + # Some systems have the definitions of the mask bits without having the + # corresponding members in struct statx. Check for members added after Linux + # 4.11 (when statx itself was added). + ac_fn_c_check_member "$LINENO" "struct statx" "stx_mnt_id" "ac_cv_member_struct_statx_stx_mnt_id" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_mnt_id" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_MNT_ID 1" >>confdefs.h + + +fi + + ac_fn_c_check_member "$LINENO" "struct statx" "stx_dio_mem_align" "ac_cv_member_struct_statx_stx_dio_mem_align" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_dio_mem_align" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN 1" >>confdefs.h + + +fi + + # stx_dio_offset_align was added together with stx_dio_mem_align + ac_fn_c_check_member "$LINENO" "struct statx" "stx_subvol" "ac_cv_member_struct_statx_stx_subvol" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_subvol" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_SUBVOL 1" >>confdefs.h + + +fi + + ac_fn_c_check_member "$LINENO" "struct statx" "stx_atomic_write_unit_min" "ac_cv_member_struct_statx_stx_atomic_write_unit_min" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_atomic_write_unit_min" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN 1" >>confdefs.h + + +fi + + # stx_atomic_write_unit_max and stx_atomic_write_segments_max were added + # together with stx_atomic_write_unit_min + ac_fn_c_check_member "$LINENO" "struct statx" "stx_dio_read_offset_align" "ac_cv_member_struct_statx_stx_dio_read_offset_align" "$ac_includes_default" +if test "x$ac_cv_member_struct_statx_stx_dio_read_offset_align" = xyes +then : + +printf "%s\n" "#define HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN 1" >>confdefs.h + + +fi + # stx_atomic_write_unit_max_opt was added in Linux 6.16, but is controlled by # the STATX_WRITE_ATOMIC mask bit added in Linux 6.11, so having the mask bit # doesn't imply having the member. diff --git a/configure.ac b/configure.ac index 65c37579633754..6f29362f46735b 100644 --- a/configure.ac +++ b/configure.ac @@ -5258,10 +5258,11 @@ AC_CHECK_FUNCS([ \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) -# Check statx availability in Linux -if test "$MACHDEP" = "linux"; then - AC_CHECK_FUNCS([statx]) -fi +# os.statx uses Linux's statx function. AIX also has a function named statx, +# but it's unrelated. Check only on Linux (including Android). +AS_CASE([$ac_sys_system], + [Linux*], [AC_CHECK_FUNCS([statx])] +) # Force lchmod off for Linux. Linux disallows changing the mode of symbolic # links. Some libc implementations have a stub lchmod implementation that always @@ -5826,6 +5827,17 @@ AC_CHECK_MEMBERS([struct passwd.pw_gecos, struct passwd.pw_passwd], [], [], [[ AC_CHECK_MEMBERS([siginfo_t.si_band], [], [], [[@%:@include ]]) if test "$ac_cv_func_statx" = yes; then + # Some systems have the definitions of the mask bits without having the + # corresponding members in struct statx. Check for members added after Linux + # 4.11 (when statx itself was added). + AC_CHECK_MEMBERS([struct statx.stx_mnt_id]) + AC_CHECK_MEMBERS([struct statx.stx_dio_mem_align]) + # stx_dio_offset_align was added together with stx_dio_mem_align + AC_CHECK_MEMBERS([struct statx.stx_subvol]) + AC_CHECK_MEMBERS([struct statx.stx_atomic_write_unit_min]) + # stx_atomic_write_unit_max and stx_atomic_write_segments_max were added + # together with stx_atomic_write_unit_min + AC_CHECK_MEMBERS([struct statx.stx_dio_read_offset_align]) # stx_atomic_write_unit_max_opt was added in Linux 6.16, but is controlled by # the STATX_WRITE_ATOMIC mask bit added in Linux 6.11, so having the mask bit # doesn't imply having the member. diff --git a/pyconfig.h.in b/pyconfig.h.in index 092f96e7152e11..fb12079bafa95e 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1334,6 +1334,23 @@ statx'. */ #undef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MAX_OPT +/* Define to 1 if 'stx_atomic_write_unit_min' is a member of 'struct statx'. + */ +#undef HAVE_STRUCT_STATX_STX_ATOMIC_WRITE_UNIT_MIN + +/* Define to 1 if 'stx_dio_mem_align' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_DIO_MEM_ALIGN + +/* Define to 1 if 'stx_dio_read_offset_align' is a member of 'struct statx'. + */ +#undef HAVE_STRUCT_STATX_STX_DIO_READ_OFFSET_ALIGN + +/* Define to 1 if 'stx_mnt_id' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_MNT_ID + +/* Define to 1 if 'stx_subvol' is a member of 'struct statx'. */ +#undef HAVE_STRUCT_STATX_STX_SUBVOL + /* Define to 1 if 'st_birthtime' is a member of 'struct stat'. */ #undef HAVE_STRUCT_STAT_ST_BIRTHTIME