Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Grammar/python.gram
Original file line number Diff line number Diff line change
Expand Up @@ -555,12 +555,12 @@ complex_number[expr_ty]:

signed_number[expr_ty]:
| NUMBER
| '+' number=NUMBER { number }
| '+' number=NUMBER { _PyAST_UnaryOp(UAdd, number, EXTRA) }
| '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) }

signed_real_number[expr_ty]:
| real_number
| '+' real=real_number { real }
| '+' real=real_number { _PyAST_UnaryOp(UAdd, real, EXTRA) }
| '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) }

real_number[expr_ty]:
Expand Down
17 changes: 15 additions & 2 deletions Lib/idlelib/idle_test/test_iomenu.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
"Test , coverage 17%."

from idlelib import iomenu
import os
import unittest
from unittest import mock
from test.support import requires
from tkinter import Tk
from tkinter import Tk, TclVersion
from idlelib.editor import EditorWindow
from idlelib import util
from idlelib import macosx, util
from idlelib.idle_test.mock_idle import Func

# Fail if either tokenize.open and t.detect_encoding does not exist.
Expand Down Expand Up @@ -38,6 +40,17 @@ def tearDownClass(cls):
def test_init(self):
self.assertIs(self.io.editwin, self.editwin)

def test_defaultfilename_tilde(self):
# gh-59568: escape a leading tilde for Tcl older than 9.
io = self.io
with mock.patch.object(io, 'filename', os.path.join('dir', '~file.py')):
dirname, base = io.defaultfilename()
self.assertEqual(dirname, 'dir')
if TclVersion < 9 and not macosx.isAquaTk():
self.assertEqual(base, os.path.join(os.curdir, '~file.py'))
else:
self.assertEqual(base, '~file.py')

def test_fixnewlines_end(self):
eq = self.assertEqual
io = self.io
Expand Down
15 changes: 15 additions & 0 deletions Lib/idlelib/idle_test/test_query.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,11 +280,20 @@ def test_blank_args(self):
dialog = self.Dummy_CustomRun(' ')
self.assertEqual(dialog.cli_args_ok(), [])

@unittest.skipIf(sys.platform == 'win32', 'not an error on Windows')
def test_invalid_args(self):
dialog = self.Dummy_CustomRun("'no-closing-quote")
self.assertEqual(dialog.cli_args_ok(), None)
self.assertIn('No closing', dialog.entry_error['text'])

@unittest.skipUnless(sys.platform == 'win32', 'Windows only')
def test_windows_args(self):
# gh-93016: backslashes are not escapes on Windows.
dialog = self.Dummy_CustomRun(r'c:\Users "c:\Program Files"')
self.assertEqual(dialog.cli_args_ok(),
[r'c:\Users', r'c:\Program Files'])
self.assertEqual(dialog.entry_error['text'], '')

def test_good_args(self):
args = ['-n', '10', '--verbose', '-p', '/path', '--name']
dialog = self.Dummy_CustomRun(' '.join(args) + ' "my name"')
Expand Down Expand Up @@ -444,6 +453,12 @@ def test_click_args(self):
dialog.entry.insert(END, ' c')
dialog.button_ok.invoke()
self.assertEqual(dialog.result, (['a', 'b=1', 'c'], True))
# gh-93016: arguments with spaces and backslashes round-trip.
args = ['a b', r'c:\dir\x']
dialog = query.CustomRun(root, 'Title', cli_args=args, _utest=True)
self.assertNotIn('{', dialog.entry.get())
dialog.button_ok.invoke()
self.assertEqual(dialog.result, (args, True))
root.destroy()


Expand Down
35 changes: 34 additions & 1 deletion Lib/idlelib/idle_test/test_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import sys
import unittest
from unittest import mock
from test.support import requires
from test.support import requires, subTests
from test.support.isolation import runInSubprocess
import tkinter
from tkinter import EventType
Expand Down Expand Up @@ -162,5 +162,38 @@ def test_fix_x11_paste(self):
self.assertEqual(after, before[cls])


class CLIargsTest(unittest.TestCase):
"Test the command line splitting and joining functions (gh-93016)."

# Expected results were verified against sys.argv of python.exe.
@subTests('cli_string,args', [
(r'c:\Users', [r'c:\Users']),
(r'\\server\share', [r'\\server\share']),
(r'"c:\Program Files\x" 1 2', [r'c:\Program Files\x', '1', '2']),
(r' x y ', ['x', 'y']),
('a\tb', ['a', 'b']),
(r'"a b"c', ['a bc']),
(r'a"b c"d', ['ab cd']),
(r'"a""b"', ['a"b']),
(r'a\"b', ['a"b']),
(r'a\\"b c" d', ['a\\b c', 'd']),
(r'a\\\"b', ['a\\"b']),
(r'"x\\"', ['x\\']),
('"c:\\Users\\"', ['c:\\Users"']),
('x\\', ['x\\']),
(r'""', ['']),
(r'"" a', ['', 'a']),
(r'"', ['']),
('', []),
])
def test_split_windows(self, cli_string, args):
self.assertEqual(util._split_windows(cli_string), args)

@subTests('args', [['a'], ['a b'], [r'c:\x'], ['q"q'], ["s's"], [''],
['\\'], ['\\"'], ['a b', r'c:\x', '', 'q"q']])
def test_split_join(self, args):
self.assertEqual(util.split_cli_args(util.join_cli_args(args)), args)


if __name__ == '__main__':
unittest.main(verbosity=2)
9 changes: 8 additions & 1 deletion Lib/idlelib/iomenu.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
import tempfile
import tokenize

from tkinter import TclVersion
from tkinter import filedialog
from tkinter import messagebox
from tkinter.simpledialog import askstring # loadfile encoding.

from idlelib.config import idleConf
from idlelib import macosx
from idlelib.util import py_extensions

py_extensions = ' '.join("*"+ext for ext in py_extensions)
Expand Down Expand Up @@ -405,7 +407,12 @@ def askopenfile(self):

def defaultfilename(self, mode="open"):
if self.filename:
return os.path.split(self.filename)
dirname, base = os.path.split(self.filename)
if base[:1] == '~' and TclVersion < 9 and not macosx.isAquaTk():
# Tcl before version 9 substitutes a leading tilde in
# a file name (gh-59568). The macOS dialog does not.
base = os.path.join(os.curdir, base)
return dirname, base
elif self.dirname:
return self.dirname, ""
else:
Expand Down
3 changes: 2 additions & 1 deletion Lib/idlelib/pyshell.py
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,8 @@ def execfile(self, filename, source=None):

def runsource(self, source):
"Extend base class method: Stuff the source in the line cache first"
# Remove the highlighting of a previous syntax error (gh-93966).
self.tkconsole.text.tag_remove("ERROR", "1.0", "end")
filename = self.stuffsource(source)
# at the moment, InteractiveInterpreter expects str
assert isinstance(source, str)
Expand Down Expand Up @@ -741,7 +743,6 @@ def showsyntaxerror(self, filename=None, **kwargs):
"""
tkconsole = self.tkconsole
text = tkconsole.text
text.tag_remove("ERROR", "1.0", "end")
type, value, tb = sys.exc_info()
if not issubclass(type, SyntaxError):
tkconsole.resetoutput()
Expand Down
11 changes: 6 additions & 5 deletions Lib/idlelib/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import importlib.util, importlib.abc
import os
import shlex
from sys import executable, platform # Platform is set for one test.

from tkinter import Toplevel, StringVar, BooleanVar, W, E, S
Expand All @@ -30,6 +29,8 @@
from tkinter.font import Font
from tkinter.simpledialog import _setup_dialog

from idlelib.util import split_cli_args, join_cli_args

class Query(Toplevel):
"""Base class for getting verified answer from a user.

Expand Down Expand Up @@ -332,6 +333,7 @@ def entry_ok(self):
path = self.path_ok()
return None if name is None or path is None else (name, path)


class CustomRun(Query):
"""Get settings for custom run of module.

Expand All @@ -344,12 +346,11 @@ def __init__(self, parent, title, *, cli_args=[],
_htest=False, _utest=False):
"""cli_args is a list of strings.

The list is assigned to the default Entry StringVar.
The strings are displayed joined by ' ' for display.
The strings are quoted and joined for display in the Entry.
"""
message = 'Command Line Arguments for sys.argv:'
super().__init__(
parent, title, message, text0=cli_args,
parent, title, message, text0=join_cli_args(cli_args),
_htest=_htest, _utest=_utest)

def create_extra(self):
Expand All @@ -369,7 +370,7 @@ def cli_args_ok(self):
"Return command line arg list or None if error."
cli_string = self.entry.get().strip()
try:
cli_args = shlex.split(cli_string, posix=True)
cli_args = split_cli_args(cli_string)
except ValueError as err:
self.showerror(str(err))
return None
Expand Down
55 changes: 55 additions & 0 deletions Lib/idlelib/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
* std streams (pyshell, run),
* warning stuff (pyshell, run).
"""
import re
import sys

# .pyw is for Windows; .pyi is for typing stub files.
Expand Down Expand Up @@ -114,6 +115,60 @@ def wheel_event(event, widget=None):
return 'break'


_cli_token_re = re.compile(r"""
(?P<backslashes>\\*)(?P<quotes>"+)
| (?P<literal>\\+|[^ \t"\\]+) # backslashes not followed by a quote
| (?P<space>[ \t]+)
""", re.VERBOSE)


def _split_windows(cli_string):
"""Split a command line into arguments as the C runtime does.

See https://learn.microsoft.com/cpp/c-language/parsing-c-command-line-arguments
"""
args = []
arg = None # None when not in an argument.
quoted = False
for m in _cli_token_re.finditer(cli_string):
match m.lastgroup:
case 'space' if not quoted:
if arg is not None:
args.append(arg)
arg = None
case 'space' | 'literal':
arg = (arg or '') + m[0]
case _: # Backslashes followed by quotes.
count = len(m['backslashes'])
escaped = count % 2 # Odd backslashes escape a quote.
bare = len(m['quotes']) - escaped
# In a quoted part every two quotes give a literal quote;
# if not quoted, the first quote opens a quoted part.
literal = escaped + ((bare + quoted - 1) // 2 if bare else 0)
arg = (arg or '') + '\\' * (count // 2) + '"' * literal
quoted ^= bare % 2
if arg is not None:
args.append(arg)
return args


def split_cli_args(cli_string): # Called in query.
"Split a command line as the Python executable does (gh-93016)."
if sys.platform == 'win32':
return _split_windows(cli_string)
import shlex
return shlex.split(cli_string)


def join_cli_args(cli_args): # Called in query.
"Join arguments into a command line which split_cli_args() splits back."
if sys.platform == 'win32':
import subprocess
return subprocess.list2cmdline(cli_args)
import shlex
return shlex.join(cli_args)


if __name__ == '__main__':
from unittest import main
main('idlelib.idle_test.test_util', verbosity=2)
28 changes: 28 additions & 0 deletions Lib/test/test_fstring.py
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,34 @@ def test_unterminated_string(self):
r"""f'{("x}'""",
])

def test_unclosed_multiline_replacement_field(self):
for prefix in ('f', 't', 'rf', 'rt'):
for quote in ('"', "'"):
triple = quote * 3
cases = (
# The apparent closing quotes open a string in the field.
('var = "abc"\na = PREFIXQUOTE {var} extern "C" { QUOTE\n'
'b = QUOTE string QUOTE', 2),
# Parentheses and dictionaries inside the field must not
# change which opening brace the diagnostic identifies.
('a = PREFIXQUOTE{\n(QUOTE', 1),
('a = PREFIXQUOTE{\n{0: QUOTE', 1),
# Use the innermost format field or formatted string.
('a = PREFIXQUOTE{0:\n{1\nQUOTE', 2),
('a = f"{\nPREFIXQUOTE{1\nQUOTE', 2),
# Account for parentheses outside the formatted string.
('a = (PREFIXQUOTE{1\nQUOTE', 1),
)
for source, lineno in cases:
source = source.replace('PREFIX', prefix).replace('QUOTE', triple)
with self.subTest(source=source):
with self.assertRaises(SyntaxError) as cm:
compile(source, '<test>', 'exec')
self.assertEqual(
cm.exception.msg,
f"{prefix[-1]}-string: expecting '}}' to close '{{' "
f"on line {lineno}")

@unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
def test_mismatched_parens(self):
self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' "
Expand Down
18 changes: 18 additions & 0 deletions Lib/test/test_unparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -693,6 +693,24 @@ def test_unparse_interactive_integrity_3(self):

self.check_src_roundtrip(src, out, mode='single')

@test.support.subTests('case', [
"case 'a string':",
"case True:",
"case _:",
"case [*_]:",
"case tuple():",
"case cls(arg, more=arg2):",
"case tuple() | list():",
"case [tuple() as obj, _, {'a': a, **more}]:",
"case -2:",
"case +2:",
"case 2 + 3j:",
"case +2 - 3j:",
])
def test_unparse_match(self, case):
src = 'match x:\n ' + case + '\n pass'
self.check_src_roundtrip(src)


class CosmeticTestCase(ASTTestCase):
"""Test if there are cosmetic issues caused by unnecessary additions"""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix AST generation asymmetry where unary positive ``+`` was implicitly dropped in pattern match expressions, restoring parsing parity with unary negative ``-``.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Include the line number of the opening brace in the :exc:`SyntaxError` for
an unclosed f-string or t-string replacement field when a matching quote
on a later line is interpreted as the start of an unterminated string.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The highlighting of a syntax error in the IDLE Shell is now removed when the
next statement is entered, not only when the next syntax error occurs.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix "Save As" and "Open" dialogs in IDLE for a file whose name starts with
``~`` on Windows and X11 with Tcl/Tk older than 9.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix parsing of command line arguments in the IDLE "Run... Customized" dialog
on Windows: backslashes are no longer treated as escape characters. Fix also
display of the previous arguments in the dialog.
14 changes: 12 additions & 2 deletions Parser/lexer/string.c
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
PyObject *res;
if (comments != NULL && comments->count > 0) {
Py_ssize_t stripped_size = expr_len;
_PyTok_Off previous_end = state->expr_span.start;

Check warning on line 81 in Parser/lexer/string.c

View workflow job for this annotation

GitHub Actions / Address sanitizer (ubuntu-26.04)

variable ‘previous_end’ set but not used [-Wunused-but-set-variable]

Check warning on line 81 in Parser/lexer/string.c

View workflow job for this annotation

GitHub Actions / Cross build Linux

variable ‘previous_end’ set but not used [-Wunused-but-set-variable]

Check warning on line 81 in Parser/lexer/string.c

View workflow job for this annotation

GitHub Actions / Ubuntu (installed) / build, install and test

variable ‘previous_end’ set but not used [-Wunused-but-set-variable]

Check warning on line 81 in Parser/lexer/string.c

View workflow job for this annotation

GitHub Actions / Address sanitizer (ubuntu-26.04)

variable ‘previous_end’ set but not used [-Wunused-but-set-variable]

Check warning on line 81 in Parser/lexer/string.c

View workflow job for this annotation

GitHub Actions / Cross build Linux

variable ‘previous_end’ set but not used [-Wunused-but-set-variable]

Check warning on line 81 in Parser/lexer/string.c

View workflow job for this annotation

GitHub Actions / Ubuntu (installed) / build, install and test

variable ‘previous_end’ set but not used [-Wunused-but-set-variable]
Py_ssize_t comment_count = 0;
for (Py_ssize_t i = 0; i < comments->count; i++) {
_PyTok_Span comment = comments->spans[i];
Expand Down Expand Up @@ -355,9 +355,19 @@

const ftstring_state *state = _PyLexer_CurrentFTString(tok);
if (state != NULL) {
/* A matching quote belongs to the surrounding formatted
* string, so the expression is missing its closing brace. */
/* A matching quote may have been intended to close the
* surrounding formatted string instead of opening a string
* inside a replacement field. */
if (state->quote == quote && state->quote_size == quote_size) {
int level = state->paren_level + state->replacement_depth - 1;
assert(level >= 0 && level < tok->level);
assert(tok->parenstack[level] == '{');
int lineno = tok->parenlinenostack[level];
if (lineno != tok->lineno) {
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
"%c-string: expecting '}' to close '{' on line %d",
_PyLexer_StringPrefix(state->kind), lineno));
}
return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
"%c-string: expecting '}'",
_PyLexer_StringPrefix(state->kind)));
Expand Down
Loading
Loading