diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 176be4ff3339555..d0fa645093a3e96 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -327,10 +327,10 @@ Functions and classes provided: .. function:: redirect_stdout(new_target) Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. + another :term:`file object`. This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. + is hardwired to :data:`sys.stdout`. For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an @@ -366,8 +366,8 @@ Functions and classes provided: .. function:: redirect_stderr(new_target) - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. + Similar to :func:`~contextlib.redirect_stdout` but redirecting the global + :data:`sys.stderr` to another :term:`file object`. This context manager is :ref:`reentrant `. diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 028a7861f36798a..1aade881745f21f 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -649,7 +649,7 @@ functions. If specified, *env* must provide any variables required for the program to execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. + specified *env* **must** include a valid ``%SystemRoot%``. .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly @@ -1473,7 +1473,7 @@ handling consistency are valid for these functions. Return ``(exitcode, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and + Execute the string *cmd* in a shell with :func:`check_output` and return a 2-tuple ``(exitcode, output)``. *encoding* and *errors* are used to decode output; see the notes on :ref:`frequently-used-arguments` for more details. diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 05ae410c168a31a..0b0dba1a996af01 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -10,12 +10,76 @@ Lexical analysis A Python program is read by a *parser*. Input to the parser is a stream of :term:`tokens `, generated by the *lexical analyzer* (also known as the *tokenizer*). -This chapter describes how the lexical analyzer breaks a file into tokens. +This chapter describes how the lexical analyzer produces these tokens. -Python reads program text as Unicode code points; the encoding of a source file -can be given by an encoding declaration and defaults to UTF-8, see :pep:`3120` -for details. If the source file cannot be decoded, a :exc:`SyntaxError` is -raised. +The lexical analyzer determines the program text's :ref:`encoding ` +(UTF-8 by default), and decodes the text into +:ref:`source characters `. +If the text cannot be decoded, a :exc:`SyntaxError` is raised. + +Next, the lexical analyzer uses the source characters to generate a stream of tokens. +The type of a generated token generally depends on the next source character to +be processed. Similarly, other special behavior of the analyzer depends on +the first source character that hasn't yet been processed. +The following table gives a quick summary of these source characters, +with links to sections that contain more information. + +.. list-table:: + :header-rows: 1 + + * - Character + - Next token (or other relevant documentation) + + * - * space + * tab + * formfeed + - * :ref:`Whitespace ` + + * - * CR, LF + - * :ref:`New line ` + * :ref:`Indentation ` + + * - * backslash (``\``) + - * :ref:`Explicit line joining ` + * (Also significant in :ref:`string escape sequences `) + + * - * hash (``#``) + - * :ref:`Comment ` + + * - * quote (``'``, ``"``) + - * :ref:`String literal ` + + * - * ASCII letter (``a``-``z``, ``A``-``Z``) + * non-ASCII character + - * :ref:`Name ` + * Prefixed :ref:`string or bytes literal ` + + * - * underscore (``_``) + - * :ref:`Name ` + * (Can also be part of :ref:`numeric literals `) + + * - * number (``0``-``9``) + - * :ref:`Numeric literal ` + + * - * dot (``.``) + - * :ref:`Numeric literal ` + * :ref:`Operator ` + + * - * question mark (``?``) + * dollar (``$``) + * + .. (the following uses zero-width space characters to render + .. a literal backquote) + + backquote (``​`​``) + * control character + - * Error (outside string literals and comments) + + * - * other printing character + - * :ref:`Operator or delimiter ` + + * - * end of file + - * :ref:`End marker ` .. _line-structure: @@ -120,6 +184,8 @@ If an encoding is declared, the encoding name must be recognized by Python encoding is used for all lexical analysis, including string literals, comments and identifiers. +.. _lexical-source-character: + All lexical analysis, including string literals, comments and identifiers, works on Unicode text decoded using the source encoding. Any Unicode code point, except the NUL control character, can appear in diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 0ee92dce43790bc..4ee09c6bbaa0691 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -35,7 +35,6 @@ Doc/library/smtplib.rst Doc/library/socket.rst Doc/library/ssl.rst Doc/library/stdtypes.rst -Doc/library/subprocess.rst Doc/library/termios.rst Doc/library/test.rst Doc/library/tkinter.rst diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 26fc9571d174233..523b505f59673e4 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -42,22 +42,22 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

{% trans %}HTML{% endtrans %} - {% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} + {% trans %}Download{% endtrans %} {% trans %}Plain text{% endtrans %} - {% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} + {% trans %}Download{% endtrans %} {% trans %}Texinfo{% endtrans %} - {% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} + {% trans %}Download{% endtrans %} {% trans %}EPUB{% endtrans %} - {% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans %}Download{% endtrans %} @@ -71,6 +71,9 @@

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

and run make dist-pdf in the Doc/ directory of a copy of the CPython repository. {% endtrans %}

+

{% trans %} +See the directory listing +for file sizes.{% endtrans %}

{% trans %}Unpacking{% endtrans %}

@@ -83,9 +86,8 @@

{% trans %}Unpacking{% endtrans %}

{% trans %}Windows users can use the ZIP archives since those are customary on that platform. These are created on Unix using the Info-ZIP zip program.{% endtrans %}

-

{% trans %}Problems{% endtrans %}

- -

{% trans %}If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.{% endtrans %}

+{% set bugs = pathto('bugs') %} +

{% trans bugs = bugs %}Open an issue +if you have comments or suggestions for the Python documentation.{% endtrans %}

{% endblock %} diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index c85c01da89a2ffb..f1c9bcd47888b10 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -11,6 +11,44 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). +#define _Py_MAX_UNICODE 0x10ffff + + +static inline void +_PyUnicode_Fill(int kind, void *data, Py_UCS4 value, + Py_ssize_t start, Py_ssize_t length) +{ + assert(0 <= start); + switch (kind) { + case PyUnicode_1BYTE_KIND: { + assert(value <= 0xff); + Py_UCS1 ch = (unsigned char)value; + Py_UCS1 *to = (Py_UCS1 *)data + start; + memset(to, ch, length); + break; + } + case PyUnicode_2BYTE_KIND: { + assert(value <= 0xffff); + Py_UCS2 ch = (Py_UCS2)value; + Py_UCS2 *to = (Py_UCS2 *)data + start; + const Py_UCS2 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + case PyUnicode_4BYTE_KIND: { + assert(value <= _Py_MAX_UNICODE); + Py_UCS4 ch = value; + Py_UCS4 * to = (Py_UCS4 *)data + start; + const Py_UCS4 *end = to + length; + for (; to < end; ++to) *to = ch; + break; + } + default: Py_UNREACHABLE(); + } +} + + /* --- Characters Type APIs ----------------------------------------------- */ extern int _PyUnicode_IsXidStart(Py_UCS4 ch); @@ -240,21 +278,6 @@ extern PyObject* _PyUnicode_XStrip( ); -/* Using explicit passed-in values, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ -extern Py_ssize_t _PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward); - /* Dedent a string. Behaviour is expected to be an exact match of `textwrap.dedent`. Return a new reference on success, NULL with exception set on error. diff --git a/Lib/argparse.py b/Lib/argparse.py index 863e951528b6d01..d71e551401cb023 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -280,7 +280,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 90bfcd0ae3e29b8..27e38040a98d10e 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -7311,11 +7311,11 @@ def custom_formatter(prog): {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} {heading}positional arguments:{reset} - {pos_b}spam{reset} spam help + {pos_b}spam{reset} spam help {heading}options:{reset} - {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit - {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help ''')) def test_custom_formatter_class(self): @@ -7348,11 +7348,11 @@ def __init__(self, prog): {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} {heading}positional arguments:{reset} - {pos_b}spam{reset} spam help + {pos_b}spam{reset} spam help {heading}options:{reset} - {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit - {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help ''')) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index d54dd546ea36fb0..e0dbb062eb0372a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2980,6 +2980,8 @@ def test_cli_converters(self): "uint64", "uint8", "unicode", + "unicode_fs_decoded", + "unicode_fs_encoded", "unsigned_char", "unsigned_int", "unsigned_long", diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 277a2a187543d47..1660dabe6815aad 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -651,6 +651,21 @@ def test_compile_filename(self): compile('pass', filename, 'exec') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') + def test_compile_filename_refleak(self): + # Regression tests for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'mode' AC conversion failed + self.assertRaises(TypeError, compile, b'', mortal_str, mode=1234) + # check error path when 'optimize' AC conversion failed + self.assertRaises(OverflowError, compile, b'', mortal_str, + 'exec', optimize=1 << 1000) + # check error path when 'dont_inherit' AC conversion failed + class EvilBool: + def __bool__(self): raise ValueError + self.assertRaises(ValueError, compile, b'', mortal_str, + 'exec', dont_inherit=EvilBool()) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 26c75fcbc2bd7f9..943e63fc13c921a 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -579,6 +579,13 @@ def test_nested_genexpr(self): self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) self.assertEqual(st.get_children(), []) + def test__symtable_refleak(self): + # Regression test for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'compile_type' AC conversion failed + self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 958a586b0dc8e86..e7931b6f394075d 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -274,7 +274,8 @@ def test_pathlike_construction(self, alpharep): """ zipfile_ondisk = self.zipfile_ondisk(alpharep) pathlike = FakePath(str(zipfile_ondisk)) - zipfile.Path(pathlike) + root = zipfile.Path(pathlike) + root.root.close() @pass_alpharep def test_traverse_pathlike(self, alpharep): @@ -373,6 +374,7 @@ def test_root_on_disk(self, alpharep): root = zipfile.Path(self.zipfile_ondisk(alpharep)) assert root.name == 'alpharep.zip' == root.filename.name assert root.stem == 'alpharep' == root.filename.stem + root.root.close() @pass_alpharep def test_suffix(self, alpharep): @@ -574,11 +576,13 @@ def test_inheritance(self, alpharep): ) def test_pickle(self, alpharep, path_type, subpath): zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) - - saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) + root = zipfile.Path(zipfile_ondisk, at=subpath) + saved_1 = pickle.dumps(root) + root.root.close() restored_1 = pickle.loads(saved_1) first, *rest = restored_1.iterdir() assert first.read_text(encoding='utf-8').startswith('content of ') + restored_1.root.close() @pass_alpharep def test_extract_orig_with_implied_dirs(self, alpharep): @@ -590,6 +594,7 @@ def test_extract_orig_with_implied_dirs(self, alpharep): # wrap the zipfile for its side effect zipfile.Path(zf) zf.extractall(source_path.parent) + zf.close() @pass_alpharep def test_getinfo_missing(self, alpharep): diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 6acfefc74d66659..1edb5dde998658f 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -312,26 +312,26 @@ def test_low_compression(self): self.assertEqual(openobj.read(1), b'2') def test_writestr_compression(self): - zipfp = zipfile.ZipFile(TESTFN2, "w") - zipfp.writestr("b.txt", "hello world", compress_type=self.compression) - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, self.compression) + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.writestr("b.txt", "hello world", compress_type=self.compression) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, self.compression) def test_writestr_compresslevel(self): - zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) - zipfp.writestr("a.txt", "hello world", compress_type=self.compression) - zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compresslevel=2) + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.writestr("a.txt", "hello world", compress_type=self.compression) + zipfp.writestr("b.txt", "hello world", compress_type=self.compression, + compresslevel=2) - # Compression level follows the constructor. - a_info = zipfp.getinfo('a.txt') - self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info.compress_level, 1) + # Compression level follows the constructor. + a_info = zipfp.getinfo('a.txt') + self.assertEqual(a_info.compress_type, self.compression) + self.assertEqual(a_info.compress_level, 1) - # Compression level is overridden. - b_info = zipfp.getinfo('b.txt') - self.assertEqual(b_info.compress_type, self.compression) - self.assertEqual(b_info._compresslevel, 2) + # Compression level is overridden. + b_info = zipfp.getinfo('b.txt') + self.assertEqual(b_info.compress_type, self.compression) + self.assertEqual(b_info._compresslevel, 2) def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes @@ -2330,6 +2330,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except zipfile.BadZipFile: self.fail("Unable to create empty ZIP file in 'w' mode") + zipf.close() zipf = zipfile.ZipFile(TESTFN, mode="a") zipf.close() @@ -2337,6 +2338,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except: self.fail("Unable to create empty ZIP file in 'a' mode") + zipf.close() def test_open_empty_file(self): # Issue 1710703: Check that opening a file with less than 22 bytes diff --git a/Makefile.pre.in b/Makefile.pre.in index 987d55a9bdbd3b5..a5223246845dcf6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -501,7 +501,6 @@ PYTHON_OBJS= \ Python/pystrtod.o \ Python/pystrhex.o \ Python/dtoa.o \ - Python/formatter_unicode.o \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ @@ -558,6 +557,7 @@ OBJECT_OBJS= \ Objects/tupleobject.o \ Objects/typeobject.o \ Objects/typevarobject.o \ + Objects/unicode_formatter.o \ Objects/unicodeobject.o \ Objects/unicodectype.o \ Objects/unionobject.o \ @@ -2091,7 +2091,6 @@ UNICODE_DEPS = \ $(srcdir)/Objects/stringlib/fastsearch.h \ $(srcdir)/Objects/stringlib/find.h \ $(srcdir)/Objects/stringlib/find_max_char.h \ - $(srcdir)/Objects/stringlib/localeutil.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/replace.h \ $(srcdir)/Objects/stringlib/repr.h \ diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst new file mode 100644 index 000000000000000..3b190f3f6df6e3b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-08-13-52-00.gh-issue-139748.jq0yFJ.rst @@ -0,0 +1,2 @@ +Fix reference leaks in error branches of functions accepting path strings or +bytes such as :func:`compile` and :func:`os.system`. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst b/Misc/NEWS.d/next/Library/2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst new file mode 100644 index 000000000000000..820679632d1e264 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-08-00-06-30.gh-issue-139736.baPeBd.rst @@ -0,0 +1,2 @@ +Fix excessive indentation in the default :mod:`argparse` +:class:`!HelpFormatter`. Patch by Alexander Edland. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 12d316985fceb93..46c4f57984b0dfc 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -15,6 +15,7 @@ #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_pyatomic_ft_wrappers.h" #include "datetime.h" @@ -2540,14 +2541,16 @@ static Py_hash_t delta_hash(PyObject *op) { PyDateTime_Delta *self = PyDelta_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *temp = delta_getstate(self); if (temp != NULL) { - self->hashcode = PyObject_Hash(temp); + hash = PyObject_Hash(temp); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp); } } - return self->hashcode; + return hash; } static PyObject * @@ -3921,12 +3924,14 @@ static Py_hash_t date_hash(PyObject *op) { PyDateTime_Date *self = PyDate_CAST(op); - if (self->hashcode == -1) { - self->hashcode = generic_hash( + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATE_DATASIZE); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); } - return self->hashcode; + return hash; } static PyObject * @@ -5043,7 +5048,8 @@ static Py_hash_t time_hash(PyObject *op) { PyDateTime_Time *self = PyTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (TIME_GET_FOLD(self)) { self0 = new_time_ex2(TIME_GET_HOUR(self), @@ -5065,10 +5071,11 @@ time_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_TIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int seconds, microseconds; assert(HASTZINFO(self)); @@ -5087,12 +5094,13 @@ time_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] @@ -6627,7 +6635,8 @@ static Py_hash_t datetime_hash(PyObject *op) { PyDateTime_DateTime *self = PyDateTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (DATE_GET_FOLD(self)) { self0 = new_datetime_ex2(GET_YEAR(self), @@ -6652,10 +6661,11 @@ datetime_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATETIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int days, seconds; @@ -6679,12 +6689,13 @@ datetime_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] diff --git a/Modules/_ssl.c b/Modules/_ssl.c index e1bdc4033ba19bf..1fa44ef1de4e093 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -1858,14 +1858,14 @@ _certificate_to_der(_sslmodulestate *state, X509 *certificate) /*[clinic input] _ssl._test_decode_cert - path: object(converter="PyUnicode_FSConverter") + path: unicode_fs_encoded / [clinic start generated code]*/ static PyObject * _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=96becb9abb23c091 input=cdeaaf02d4346628]*/ +/*[clinic end generated code: output=96becb9abb23c091 input=cb4988d5e651a4f8]*/ { PyObject *retval = NULL; X509 *x=NULL; @@ -1895,7 +1895,6 @@ _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) X509_free(x); fail0: - Py_DECREF(path); if (cert != NULL) BIO_free(cert); return retval; } diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 0e8e4a7c8a73c73..d1fb024903e1572 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -48,7 +48,7 @@ static PyObject * _ssl__test_decode_cert(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *path; + PyObject *path = NULL; if (!PyUnicode_FSConverter(arg, &path)) { goto exit; @@ -56,6 +56,9 @@ _ssl__test_decode_cert(PyObject *module, PyObject *arg) return_value = _ssl__test_decode_cert_impl(module, path); exit: + /* Cleanup for path */ + Py_XDECREF(path); + return return_value; } @@ -3322,4 +3325,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=5a630a1e83927d47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3b6c9cbfc4660ecb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 0cedab597db714c..e0cc1c50dcbac30 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -479,7 +479,7 @@ static PyObject * _socket_if_nametoindex(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *oname; + PyObject *oname = NULL; if (!PyUnicode_FSConverter(arg, &oname)) { goto exit; @@ -487,6 +487,9 @@ _socket_if_nametoindex(PyObject *module, PyObject *arg) return_value = _socket_if_nametoindex_impl(module, oname); exit: + /* Cleanup for oname */ + Py_XDECREF(oname); + return return_value; } @@ -538,4 +541,4 @@ _socket_if_indextoname(PyObject *module, PyObject *arg) #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=0376c46b76ae2bce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=36051ebf6ad1e6f8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 2ecd3afc00d2be4..bd55d77c5409e93 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -22,7 +22,7 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *source; - PyObject *filename; + PyObject *filename = NULL; const char *startstr; if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { @@ -48,6 +48,9 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return_value = _symtable_symtable_impl(module, source, filename, startstr); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } -/*[clinic end generated code: output=931964a76a72f850 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7a8545d9a1efe837 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f50167c223e2fc4..4189d300856068b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3137,17 +3137,6 @@ class dev_t_return_converter(unsigned_long_return_converter): conversion_fn = '_PyLong_FromDev' unsigned_cast = '(dev_t)' -class FSConverter_converter(CConverter): - type = 'PyObject *' - converter = 'PyUnicode_FSConverter' - def converter_init(self): - if self.default is not unspecified: - fail("FSConverter_converter does not support default values") - self.c_default = 'NULL' - - def cleanup(self): - return "Py_XDECREF(" + self.name + ");\n" - class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' @@ -3211,7 +3200,7 @@ class confname_converter(CConverter): """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=8189d5ae78244626]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=d2759f2332cd39b3]*/ /*[clinic input] @@ -6135,14 +6124,14 @@ os_system_impl(PyObject *module, const wchar_t *command) /*[clinic input] os.system -> long - command: FSConverter + command: unicode_fs_encoded Execute the command in a subshell. [clinic start generated code]*/ static long os_system_impl(PyObject *module, PyObject *command) -/*[clinic end generated code: output=290fc437dd4f33a0 input=86a58554ba6094af]*/ +/*[clinic end generated code: output=290fc437dd4f33a0 input=47c6f24b6dc92881]*/ { long result; const char *bytes = PyBytes_AsString(command); @@ -9328,7 +9317,7 @@ os_getgroups_impl(PyObject *module) /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: int / @@ -9341,12 +9330,12 @@ group id. static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, int gid) -/*[clinic end generated code: output=7f074d30a425fd3a input=df3d54331b0af204]*/ +/*[clinic end generated code: output=7f074d30a425fd3a input=984e60c7fed88cb4]*/ #else /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: gid_t / @@ -9359,7 +9348,7 @@ group id. static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, gid_t gid) -/*[clinic end generated code: output=59341244521a9e3f input=0cb91bdc59a4c564]*/ +/*[clinic end generated code: output=59341244521a9e3f input=17d8fbe2dea42ca4]*/ #endif { const char *username = PyBytes_AS_STRING(oname); @@ -13115,8 +13104,8 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) /*[clinic input] os.putenv - name: FSConverter - value: FSConverter + name: unicode_fs_encoded + value: unicode_fs_encoded / Change or add an environment variable. @@ -13124,7 +13113,7 @@ Change or add an environment variable. static PyObject * os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) -/*[clinic end generated code: output=d29a567d6b2327d2 input=a97bc6152f688d31]*/ +/*[clinic end generated code: output=d29a567d6b2327d2 input=84fcd30f873c8c45]*/ { const char *name_string = PyBytes_AS_STRING(name); const char *value_string = PyBytes_AS_STRING(value); @@ -13167,7 +13156,7 @@ os_unsetenv_impl(PyObject *module, PyObject *name) #else /*[clinic input] os.unsetenv - name: FSConverter + name: unicode_fs_encoded / Delete an environment variable. @@ -13175,7 +13164,7 @@ Delete an environment variable. static PyObject * os_unsetenv_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=54c4137ab1834f02 input=2bb5288a599c7107]*/ +/*[clinic end generated code: output=54c4137ab1834f02 input=78ff12e505ade80a]*/ { if (PySys_Audit("os.unsetenv", "(O)", name) < 0) { return NULL; @@ -15229,14 +15218,14 @@ os_urandom_impl(PyObject *module, Py_ssize_t size) /*[clinic input] os.memfd_create - name: FSConverter + name: unicode_fs_encoded flags: unsigned_int(bitwise=True, c_default="MFD_CLOEXEC") = MFD_CLOEXEC [clinic start generated code]*/ static PyObject * os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) -/*[clinic end generated code: output=6681ede983bdb9a6 input=a42cfc199bcd56e9]*/ +/*[clinic end generated code: output=6681ede983bdb9a6 input=cd0eb092cfac474b]*/ { int fd; const char *bytes = PyBytes_AS_STRING(name); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index fa153efd8b72ec0..dd4b68929774843 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7278,7 +7278,7 @@ Returns a list of network interface information (index, name) tuples."); /*[clinic input] _socket.if_nametoindex - oname: object(converter="PyUnicode_FSConverter") + oname: unicode_fs_encoded / Returns the interface index corresponding to the interface name if_name. @@ -7286,7 +7286,7 @@ Returns the interface index corresponding to the interface name if_name. static PyObject * _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) -/*[clinic end generated code: output=289a411614f30244 input=01e0f1205307fb77]*/ +/*[clinic end generated code: output=289a411614f30244 input=6125dc20683560cf]*/ { #ifdef MS_WINDOWS NET_IFINDEX index; @@ -7295,7 +7295,6 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) #endif index = if_nametoindex(PyBytes_AS_STRING(oname)); - Py_DECREF(oname); if (index == 0) { /* if_nametoindex() doesn't set errno */ PyErr_SetString(PyExc_OSError, "no interface with this name"); diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d0d5223e5acea85..d353f406831ecd2 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -13,7 +13,7 @@ module _symtable _symtable.symtable source: object - filename: object(converter='PyUnicode_FSDecoder') + filename: unicode_fs_decoded startstr: str / @@ -23,7 +23,7 @@ Return symbol and scope dictionaries used internally by compiler. static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/ +/*[clinic end generated code: output=59eb0d5fc7285ac4 input=436ffff90d02e4f6]*/ { struct symtable *st; PyObject *t; @@ -47,12 +47,10 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, else { PyErr_SetString(PyExc_ValueError, "symtable() arg 3 must be 'exec' or 'eval' or 'single'"); - Py_DECREF(filename); Py_XDECREF(source_copy); return NULL; } st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); - Py_DECREF(filename); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Objects/stringlib/localeutil.h b/Objects/stringlib/localeutil.h deleted file mode 100644 index a4ab701de004c85..000000000000000 --- a/Objects/stringlib/localeutil.h +++ /dev/null @@ -1,97 +0,0 @@ -/* _PyUnicode_InsertThousandsGrouping() helper functions */ - -typedef struct { - const char *grouping; - char previous; - Py_ssize_t i; /* Where we're currently pointing in grouping. */ -} GroupGenerator; - - -static void -GroupGenerator_init(GroupGenerator *self, const char *grouping) -{ - self->grouping = grouping; - self->i = 0; - self->previous = 0; -} - - -/* Returns the next grouping, or 0 to signify end. */ -static Py_ssize_t -GroupGenerator_next(GroupGenerator *self) -{ - /* Note that we don't really do much error checking here. If a - grouping string contains just CHAR_MAX, for example, then just - terminate the generator. That shouldn't happen, but at least we - fail gracefully. */ - switch (self->grouping[self->i]) { - case 0: - return self->previous; - case CHAR_MAX: - /* Stop the generator. */ - return 0; - default: { - char ch = self->grouping[self->i]; - self->previous = ch; - self->i++; - return (Py_ssize_t)ch; - } - } -} - - -/* Fill in some digits, leading zeros, and thousands separator. All - are optional, depending on when we're called. */ -static void -InsertThousandsGrouping_fill(_PyUnicodeWriter *writer, Py_ssize_t *buffer_pos, - PyObject *digits, Py_ssize_t *digits_pos, - Py_ssize_t n_chars, Py_ssize_t n_zeros, - PyObject *thousands_sep, Py_ssize_t thousands_sep_len, - Py_UCS4 *maxchar, int forward) -{ - if (!writer) { - /* if maxchar > 127, maxchar is already set */ - if (*maxchar == 127 && thousands_sep) { - Py_UCS4 maxchar2 = PyUnicode_MAX_CHAR_VALUE(thousands_sep); - *maxchar = Py_MAX(*maxchar, maxchar2); - } - return; - } - - if (thousands_sep) { - if (!forward) { - *buffer_pos -= thousands_sep_len; - } - /* Copy the thousands_sep chars into the buffer. */ - _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, - thousands_sep, 0, - thousands_sep_len); - if (forward) { - *buffer_pos += thousands_sep_len; - } - } - - if (!forward) { - *buffer_pos -= n_chars; - *digits_pos -= n_chars; - } - _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, - digits, *digits_pos, - n_chars); - if (forward) { - *buffer_pos += n_chars; - *digits_pos += n_chars; - } - - if (n_zeros) { - if (!forward) { - *buffer_pos -= n_zeros; - } - int kind = PyUnicode_KIND(writer->buffer); - void *data = PyUnicode_DATA(writer->buffer); - unicode_fill(kind, data, '0', *buffer_pos, n_zeros); - if (forward) { - *buffer_pos += n_zeros; - } - } -} diff --git a/Python/formatter_unicode.c b/Objects/unicode_formatter.c similarity index 88% rename from Python/formatter_unicode.c rename to Objects/unicode_formatter.c index 30807f428c7d71b..b8604d1355940a5 100644 --- a/Python/formatter_unicode.c +++ b/Objects/unicode_formatter.c @@ -8,6 +8,241 @@ #include "pycore_unicodeobject.h" // PyUnicode_MAX_CHAR_VALUE() #include + +/* _PyUnicode_InsertThousandsGrouping() helper functions */ + +typedef struct { + const char *grouping; + char previous; + Py_ssize_t i; /* Where we're currently pointing in grouping. */ +} GroupGenerator; + + +static void +GroupGenerator_init(GroupGenerator *self, const char *grouping) +{ + self->grouping = grouping; + self->i = 0; + self->previous = 0; +} + + +/* Returns the next grouping, or 0 to signify end. */ +static Py_ssize_t +GroupGenerator_next(GroupGenerator *self) +{ + /* Note that we don't really do much error checking here. If a + grouping string contains just CHAR_MAX, for example, then just + terminate the generator. That shouldn't happen, but at least we + fail gracefully. */ + switch (self->grouping[self->i]) { + case 0: + return self->previous; + case CHAR_MAX: + /* Stop the generator. */ + return 0; + default: { + char ch = self->grouping[self->i]; + self->previous = ch; + self->i++; + return (Py_ssize_t)ch; + } + } +} + + +/* Fill in some digits, leading zeros, and thousands separator. All + are optional, depending on when we're called. */ +static void +InsertThousandsGrouping_fill(_PyUnicodeWriter *writer, Py_ssize_t *buffer_pos, + PyObject *digits, Py_ssize_t *digits_pos, + Py_ssize_t n_chars, Py_ssize_t n_zeros, + PyObject *thousands_sep, Py_ssize_t thousands_sep_len, + Py_UCS4 *maxchar, int forward) +{ + if (!writer) { + /* if maxchar > 127, maxchar is already set */ + if (*maxchar == 127 && thousands_sep) { + Py_UCS4 maxchar2 = PyUnicode_MAX_CHAR_VALUE(thousands_sep); + *maxchar = Py_MAX(*maxchar, maxchar2); + } + return; + } + + if (thousands_sep) { + if (!forward) { + *buffer_pos -= thousands_sep_len; + } + /* Copy the thousands_sep chars into the buffer. */ + _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, + thousands_sep, 0, + thousands_sep_len); + if (forward) { + *buffer_pos += thousands_sep_len; + } + } + + if (!forward) { + *buffer_pos -= n_chars; + *digits_pos -= n_chars; + } + _PyUnicode_FastCopyCharacters(writer->buffer, *buffer_pos, + digits, *digits_pos, + n_chars); + if (forward) { + *buffer_pos += n_chars; + *digits_pos += n_chars; + } + + if (n_zeros) { + if (!forward) { + *buffer_pos -= n_zeros; + } + int kind = PyUnicode_KIND(writer->buffer); + void *data = PyUnicode_DATA(writer->buffer); + _PyUnicode_Fill(kind, data, '0', *buffer_pos, n_zeros); + if (forward) { + *buffer_pos += n_zeros; + } + } +} + + +/** + * InsertThousandsGrouping: + * @writer: Unicode writer. + * @n_buffer: Number of characters in @buffer. + * @digits: Digits we're reading from. If count is non-NULL, this is unused. + * @d_pos: Start of digits string. + * @n_digits: The number of digits in the string, in which we want + * to put the grouping chars. + * @min_width: The minimum width of the digits in the output string. + * Output will be zero-padded on the left to fill. + * @grouping: see definition in localeconv(). + * @thousands_sep: see definition in localeconv(). + * + * There are 2 modes: counting and filling. If @writer is NULL, + * we are in counting mode, else filling mode. + * If counting, the required buffer size is returned. + * If filling, we know the buffer will be large enough, so we don't + * need to pass in the buffer size. + * Inserts thousand grouping characters (as defined by grouping and + * thousands_sep) into @writer. + * + * Return value: -1 on error, number of characters otherwise. + **/ +static Py_ssize_t +_PyUnicode_InsertThousandsGrouping( + _PyUnicodeWriter *writer, + Py_ssize_t n_buffer, + PyObject *digits, + Py_ssize_t d_pos, + Py_ssize_t n_digits, + Py_ssize_t min_width, + const char *grouping, + PyObject *thousands_sep, + Py_UCS4 *maxchar, + int forward) +{ + min_width = Py_MAX(0, min_width); + if (writer) { + assert(digits != NULL); + assert(maxchar == NULL); + } + else { + assert(digits == NULL); + assert(maxchar != NULL); + } + assert(0 <= d_pos); + assert(0 <= n_digits); + assert(grouping != NULL); + + Py_ssize_t count = 0; + Py_ssize_t n_zeros; + int loop_broken = 0; + int use_separator = 0; /* First time through, don't append the + separator. They only go between + groups. */ + Py_ssize_t buffer_pos; + Py_ssize_t digits_pos; + Py_ssize_t len; + Py_ssize_t n_chars; + Py_ssize_t remaining = n_digits; /* Number of chars remaining to + be looked at */ + /* A generator that returns all of the grouping widths, until it + returns 0. */ + GroupGenerator groupgen; + GroupGenerator_init(&groupgen, grouping); + const Py_ssize_t thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); + + /* if digits are not grouped, thousands separator + should be an empty string */ + assert(!(grouping[0] == CHAR_MAX && thousands_sep_len != 0)); + + digits_pos = d_pos + (forward ? 0 : n_digits); + if (writer) { + buffer_pos = writer->pos + (forward ? 0 : n_buffer); + assert(buffer_pos <= PyUnicode_GET_LENGTH(writer->buffer)); + assert(digits_pos <= PyUnicode_GET_LENGTH(digits)); + } + else { + buffer_pos = forward ? 0 : n_buffer; + } + + if (!writer) { + *maxchar = 127; + } + + while ((len = GroupGenerator_next(&groupgen)) > 0) { + len = Py_MIN(len, Py_MAX(Py_MAX(remaining, min_width), 1)); + n_zeros = Py_MAX(0, len - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, len)); + + /* Use n_zero zero's and n_chars chars */ + + /* Count only, don't do anything. */ + count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; + + /* Copy into the writer. */ + InsertThousandsGrouping_fill(writer, &buffer_pos, + digits, &digits_pos, + n_chars, n_zeros, + use_separator ? thousands_sep : NULL, + thousands_sep_len, maxchar, forward); + + /* Use a separator next time. */ + use_separator = 1; + + remaining -= n_chars; + min_width -= len; + + if (remaining <= 0 && min_width <= 0) { + loop_broken = 1; + break; + } + min_width -= thousands_sep_len; + } + if (!loop_broken) { + /* We left the loop without using a break statement. */ + + len = Py_MAX(Py_MAX(remaining, min_width), 1); + n_zeros = Py_MAX(0, len - remaining); + n_chars = Py_MAX(0, Py_MIN(remaining, len)); + + /* Use n_zero zero's and n_chars chars */ + count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; + + /* Copy into the writer. */ + InsertThousandsGrouping_fill(writer, &buffer_pos, + digits, &digits_pos, + n_chars, n_zeros, + use_separator ? thousands_sep : NULL, + thousands_sep_len, maxchar, forward); + } + return count; +} + + /* Raises an exception about an unknown presentation type for this * type. */ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 11ba147a7446923..c71f9d3f71dea5e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -104,9 +104,7 @@ NOTE: In the interpreter's initialization phase, some globals are currently */ -// Maximum code point of Unicode 6.0: 0x10ffff (1,114,111). -// The value must be the same in fileutils.c. -#define MAX_UNICODE 0x10ffff +#define MAX_UNICODE _Py_MAX_UNICODE #ifdef Py_DEBUG # define _PyUnicode_CHECK(op) _PyUnicode_CheckConsistency(op, 0) @@ -420,39 +418,6 @@ static void clear_global_interned_strings(void) return unicode_get_empty(); \ } while (0) -static inline void -unicode_fill(int kind, void *data, Py_UCS4 value, - Py_ssize_t start, Py_ssize_t length) -{ - assert(0 <= start); - switch (kind) { - case PyUnicode_1BYTE_KIND: { - assert(value <= 0xff); - Py_UCS1 ch = (unsigned char)value; - Py_UCS1 *to = (Py_UCS1 *)data + start; - memset(to, ch, length); - break; - } - case PyUnicode_2BYTE_KIND: { - assert(value <= 0xffff); - Py_UCS2 ch = (Py_UCS2)value; - Py_UCS2 *to = (Py_UCS2 *)data + start; - const Py_UCS2 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - case PyUnicode_4BYTE_KIND: { - assert(value <= MAX_UNICODE); - Py_UCS4 ch = value; - Py_UCS4 * to = (Py_UCS4 *)data + start; - const Py_UCS4 *end = to + length; - for (; to < end; ++to) *to = ch; - break; - } - default: Py_UNREACHABLE(); - } -} - /* Fast detection of the most frequent whitespace characters */ const unsigned char _Py_ascii_whitespace[] = { @@ -9735,142 +9700,6 @@ any_find_slice(PyObject* s1, PyObject* s2, return result; } -/* _PyUnicode_InsertThousandsGrouping() helper functions */ -#include "stringlib/localeutil.h" - -/** - * InsertThousandsGrouping: - * @writer: Unicode writer. - * @n_buffer: Number of characters in @buffer. - * @digits: Digits we're reading from. If count is non-NULL, this is unused. - * @d_pos: Start of digits string. - * @n_digits: The number of digits in the string, in which we want - * to put the grouping chars. - * @min_width: The minimum width of the digits in the output string. - * Output will be zero-padded on the left to fill. - * @grouping: see definition in localeconv(). - * @thousands_sep: see definition in localeconv(). - * - * There are 2 modes: counting and filling. If @writer is NULL, - * we are in counting mode, else filling mode. - * If counting, the required buffer size is returned. - * If filling, we know the buffer will be large enough, so we don't - * need to pass in the buffer size. - * Inserts thousand grouping characters (as defined by grouping and - * thousands_sep) into @writer. - * - * Return value: -1 on error, number of characters otherwise. - **/ -Py_ssize_t -_PyUnicode_InsertThousandsGrouping( - _PyUnicodeWriter *writer, - Py_ssize_t n_buffer, - PyObject *digits, - Py_ssize_t d_pos, - Py_ssize_t n_digits, - Py_ssize_t min_width, - const char *grouping, - PyObject *thousands_sep, - Py_UCS4 *maxchar, - int forward) -{ - min_width = Py_MAX(0, min_width); - if (writer) { - assert(digits != NULL); - assert(maxchar == NULL); - } - else { - assert(digits == NULL); - assert(maxchar != NULL); - } - assert(0 <= d_pos); - assert(0 <= n_digits); - assert(grouping != NULL); - - Py_ssize_t count = 0; - Py_ssize_t n_zeros; - int loop_broken = 0; - int use_separator = 0; /* First time through, don't append the - separator. They only go between - groups. */ - Py_ssize_t buffer_pos; - Py_ssize_t digits_pos; - Py_ssize_t len; - Py_ssize_t n_chars; - Py_ssize_t remaining = n_digits; /* Number of chars remaining to - be looked at */ - /* A generator that returns all of the grouping widths, until it - returns 0. */ - GroupGenerator groupgen; - GroupGenerator_init(&groupgen, grouping); - const Py_ssize_t thousands_sep_len = PyUnicode_GET_LENGTH(thousands_sep); - - /* if digits are not grouped, thousands separator - should be an empty string */ - assert(!(grouping[0] == CHAR_MAX && thousands_sep_len != 0)); - - digits_pos = d_pos + (forward ? 0 : n_digits); - if (writer) { - buffer_pos = writer->pos + (forward ? 0 : n_buffer); - assert(buffer_pos <= PyUnicode_GET_LENGTH(writer->buffer)); - assert(digits_pos <= PyUnicode_GET_LENGTH(digits)); - } - else { - buffer_pos = forward ? 0 : n_buffer; - } - - if (!writer) { - *maxchar = 127; - } - - while ((len = GroupGenerator_next(&groupgen)) > 0) { - len = Py_MIN(len, Py_MAX(Py_MAX(remaining, min_width), 1)); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); - - /* Use n_zero zero's and n_chars chars */ - - /* Count only, don't do anything. */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; - - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar, forward); - - /* Use a separator next time. */ - use_separator = 1; - - remaining -= n_chars; - min_width -= len; - - if (remaining <= 0 && min_width <= 0) { - loop_broken = 1; - break; - } - min_width -= thousands_sep_len; - } - if (!loop_broken) { - /* We left the loop without using a break statement. */ - - len = Py_MAX(Py_MAX(remaining, min_width), 1); - n_zeros = Py_MAX(0, len - remaining); - n_chars = Py_MAX(0, Py_MIN(remaining, len)); - - /* Use n_zero zero's and n_chars chars */ - count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars; - - /* Copy into the writer. */ - InsertThousandsGrouping_fill(writer, &buffer_pos, - digits, &digits_pos, - n_chars, n_zeros, - use_separator ? thousands_sep : NULL, - thousands_sep_len, maxchar, forward); - } - return count; -} Py_ssize_t PyUnicode_Count(PyObject *str, @@ -10427,7 +10256,7 @@ _PyUnicode_FastFill(PyObject *unicode, Py_ssize_t start, Py_ssize_t length, assert(fill_char <= PyUnicode_MAX_CHAR_VALUE(unicode)); assert(start >= 0); assert(start + length <= PyUnicode_GET_LENGTH(unicode)); - unicode_fill(kind, data, fill_char, start, length); + _PyUnicode_Fill(kind, data, fill_char, start, length); } Py_ssize_t @@ -10496,9 +10325,10 @@ pad(PyObject *self, kind = PyUnicode_KIND(u); data = PyUnicode_DATA(u); if (left) - unicode_fill(kind, data, fill, 0, left); + _PyUnicode_Fill(kind, data, fill, 0, left); if (right) - unicode_fill(kind, data, fill, left + _PyUnicode_LENGTH(self), right); + _PyUnicode_Fill(kind, data, fill, + left + _PyUnicode_LENGTH(self), right); _PyUnicode_FastCopyCharacters(u, left, self, 0, _PyUnicode_LENGTH(self)); assert(_PyUnicode_CheckConsistency(u, 1)); return u; @@ -11910,7 +11740,7 @@ unicode_expandtabs_impl(PyObject *self, int tabsize) if (tabsize > 0) { incr = tabsize - (line_pos % tabsize); line_pos += incr; - unicode_fill(kind, dest_data, ' ', j, incr); + _PyUnicode_Fill(kind, dest_data, ' ', j, incr); j += incr; } } @@ -15405,7 +15235,7 @@ unicode_format_arg_output(struct unicode_formatter_t *ctx, /* Pad left with the fill character if needed */ if (arg->width > len && !(arg->flags & F_LJUST)) { sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, fill, writer->pos, sublen); + _PyUnicode_Fill(writer->kind, writer->data, fill, writer->pos, sublen); writer->pos += sublen; arg->width = len; } @@ -15437,7 +15267,7 @@ unicode_format_arg_output(struct unicode_formatter_t *ctx, /* Pad right with the fill character if needed */ if (arg->width > len) { sublen = arg->width - len; - unicode_fill(writer->kind, writer->data, ' ', writer->pos, sublen); + _PyUnicode_Fill(writer->kind, writer->data, ' ', writer->pos, sublen); writer->pos += sublen; } return 0; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 5ceddf759b8f3bc..c4a11fa9b242bdf 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -165,6 +165,7 @@ + @@ -209,7 +210,6 @@ - diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 332d466b1f7409b..7bbbec2c9887bf3 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -160,9 +160,6 @@ Source Files - - Source Files - Source Files @@ -487,6 +484,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d65d080ca71a901..841d83968c60be8 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -47,7 +47,7 @@ @rem If py.exe finds a recent enough version, use that one @rem It is fine to add new versions to this list when they have released, @rem but we do not use prerelease builds here. -@for %%p in (3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.14 3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 71f508a7e8b5fbd..e2e1e415827e6f6 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -559,6 +559,7 @@ + @@ -605,7 +606,6 @@ - $(GeneratedFrozenModulesDir)Python;%(AdditionalIncludeDirectories) diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 547e9ee1abf3704..7e7ed9c2ae6c439 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1274,6 +1274,9 @@ Objects + + Objects + Objects @@ -1382,9 +1385,6 @@ Python - - Python - Python diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index b7e08ae54da578f..2551c2c961b6431 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -745,7 +745,7 @@ builtin_chr(PyObject *module, PyObject *i) compile as builtin_compile source: object - filename: object(converter="PyUnicode_FSDecoder") + filename: unicode_fs_decoded mode: str flags: int = 0 dont_inherit: bool = False @@ -771,7 +771,7 @@ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=cc78e20e7c7682ba]*/ +/*[clinic end generated code: output=b0c09c84f116d3d7 input=8f0069edbdac381b]*/ { PyObject *source_copy; const char *str; @@ -889,7 +889,6 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, error: result = NULL; finally: - Py_DECREF(filename); return result; } diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 60e20a46af5b93c..adb82f45c25b5d1 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -296,7 +296,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[7]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; - PyObject *filename; + PyObject *filename = NULL; const char *mode; int flags = 0; int dont_inherit = 0; @@ -367,6 +367,9 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } @@ -1274,4 +1277,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c0b72519622c849e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7eada753dc2e046f input=a9049054013a1b77]*/ diff --git a/Python/fileutils.c b/Python/fileutils.c index 2a3f12d4e872f89..b808229716fd9ca 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2,6 +2,7 @@ #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "pycore_pystate.h" // _Py_AssertHoldsTstate() +#include "pycore_unicodeobject.h" // _Py_MAX_UNICODE #include "osdefs.h" // SEP #include // mbstowcs() @@ -50,9 +51,6 @@ extern int winerror_to_errno(int); int _Py_open_cloexec_works = -1; #endif -// The value must be the same in unicodeobject.c. -#define MAX_UNICODE 0x10ffff - // mbstowcs() and mbrtowc() errors static const size_t DECODE_ERROR = ((size_t)-1); #ifdef HAVE_MBRTOWC @@ -123,7 +121,7 @@ is_valid_wide_char(wchar_t ch) { #ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION /* Oracle Solaris doesn't use Unicode code points as wchar_t encoding - for non-Unicode locales, which makes values higher than MAX_UNICODE + for non-Unicode locales, which makes values higher than _Py_MAX_UNICODE possibly valid. */ return 1; #endif @@ -132,7 +130,7 @@ is_valid_wide_char(wchar_t ch) return 0; } #if SIZEOF_WCHAR_T > 2 - if (ch > MAX_UNICODE) { + if (ch > _Py_MAX_UNICODE) { // bpo-35883: Reject characters outside [U+0000; U+10ffff] range. // The glibc mbstowcs() UTF-8 decoder does not respect the RFC 3629, // it creates characters outside the [U+0000; U+10ffff] range: diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 3154299e31b4dce..201125b9165e748 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -916,6 +916,26 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) +class _unicode_fs_converter_base(CConverter): + type = 'PyObject *' + + def converter_init(self) -> None: + if self.default is not unspecified: + fail(f"{self.__class__.__name__} does not support default values") + self.c_default = 'NULL' + + def cleanup(self) -> str: + return f"Py_XDECREF({self.parser_name});" + + +class unicode_fs_encoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSConverter' + + +class unicode_fs_decoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSDecoder' + + @add_legacy_c_converter('u') @add_legacy_c_converter('u#', zeroes=True) @add_legacy_c_converter('Z', accept={str, NoneType})