PEP 822: d-string draft implementation

Lib/test/test_dstring.py

@@ -0,0 +1,264 @@
+import unittest
+
+from test.test_string._support import TStringBaseCase
+
+
+_dstring_prefixes = "d db df dt dr drb drf drt".split()
+_dstring_prefixes += [p.upper() for p in _dstring_prefixes]
+
+
+def d(s):
+    """Helper function to evaluate d-strings."""
+    if '"""' in s:
+        return eval(f"d'''{s}'''")
+    else:
+        return eval(f'd"""{s}"""')
+
+def db(s):
+    """Helper function to evaluate db-strings."""
+    if '"""' in s:
+        return eval(f"db'''{s}'''")
+    else:
+        return eval(f'db"""{s}"""')
+
+def fd(s, globals=None):
+    """Helper function to evaluate fd-strings."""
+    if '"""' in s:
+        return eval(f"fd'''{s}'''", globals=globals)
+    else:
+        return eval(f'fd"""{s}"""', globals=globals)
+
+
+
+class DStringTestCase(unittest.TestCase):
+    def assertAllRaise(self, exception_type, regex, error_strings):
+        for str in error_strings:
+            with self.subTest(str=str):
+                with self.assertRaisesRegex(exception_type, regex) as cm:
+                    eval(str)
+
+    def test_single_quote(self):
+        exprs = [
+            f"{p}'hello, world'" for p in _dstring_prefixes
+        ] + [
+            f'{p}"hello, world"' for p in _dstring_prefixes
+        ]
+        self.assertAllRaise(SyntaxError, "d-string must be triple-quoted", exprs)
+
+    def test_empty_dstring(self):
+        exprs = [
+            f"{p}''''''" for p in _dstring_prefixes
+        ] + [
+            f'{p}""""""' for p in _dstring_prefixes
+        ]
+        self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs)
+
+        for prefix in _dstring_prefixes:
+            expr = f"{prefix}'''\n'''"
+            expr2 = f'{prefix}"""\n"""'
+            with self.subTest(expr=expr):
+                v = eval(expr)
+                v2 = eval(expr2)
+                if 't' in prefix.lower():
+                    self.assertEqual(v.strings, ("",))
+                    self.assertEqual(v2.strings, ("",))
+                elif 'b' in prefix.lower():
+                    self.assertEqual(v, b"")
+                    self.assertEqual(v2, b"")
+                else:
+                    self.assertEqual(v, "")
+                    self.assertEqual(v2, "")
+
+    def test_missing_newline_in_plain_and_raw_prefixes(self):
+        exprs = [
+            'd"""x"""',
+            'dr"""x"""',
+            'db"""x"""',
+            'drb"""x"""',
+        ]
+        self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs)
+
+    def check_dbstring(self, s, expected):
+        self.assertEqual(d(s), expected)
+        self.assertEqual(db(s), expected.encode())
+
+    def test_dbstring(self):
+        # Basic dedent - remove common leading whitespace
+        source = """
+            hello
+            world
+            """
+        self.check_dbstring(source, "hello\nworld\n")
+
+        # closing quote on same line as last content line
+        source = """
+            hello
+            world"""
+        self.check_dbstring(source, "hello\nworld")
+
+        # Dedent with varying indentation
+        source = """
+            .line1
+            ...line2
+            line3
+            ..""".replace('.', ' ')
+        self.check_dbstring(source, " line1\n   line2\nline3\n  ")
+
+        # Dedent with tabs
+        source = """
+\thello
+\tworld
+\t"""
+        self.check_dbstring(source, "hello\nworld\n")
+
+        # Mixed spaces and tabs (using common leading whitespace)
+        source = """
+\t\t    hello
+\t\t    world
+\t\t  """
+        self.check_dbstring(source, "  hello\n  world\n")
+
+        # Empty lines do not affect the calculation of common leading whitespace
+        source = """
+    hello
+
+    world
+    """
+        self.check_dbstring(source, "hello\n\nworld\n")
+
+        # Lines with only whitespace also have their indentation removed.
+        source = """
+....hello
+..
+......
+....world
+....""".replace('.', ' ')
+        self.check_dbstring(source, "hello\n\n  \nworld\n")
+
+        # Line continuation with backslash works as usual.
+        # But you cannot put a backslash right after the opening quotes.
+        source = r"""
+            Hello \
+            World!\
+            """
+        self.check_dbstring(source, "Hello World!")
+
+    def test_raw_dstring(self):
+        source = r"""
+            path\\to\\file
+            keep\\n
+            """
+        self.assertEqual(d(source), "path\\to\\file\nkeep\\n\n")
+
+    def test_raw_dbstring(self):
+        source = r"""
+            path\\to\\file
+            keep\\n
+            """
+        self.assertEqual(db(source), b"path\\to\\file\nkeep\\n\n")
+
+    def test_dbstring_non_ascii_error(self):
+        with self.assertRaisesRegex(SyntaxError, "bytes can only contain ASCII literal characters"):
+            eval('db"""\n  \u00e9\n  """')
+
+    def test_concat_bytes_and_nonbytes_error(self):
+        exprs = [
+            'd"""\n    x\n    """ db"""\n    y\n    """',
+            'db"""\n    x\n    """ d"""\n    y\n    """',
+        ]
+        self.assertAllRaise(SyntaxError, "cannot mix bytes and nonbytes literals", exprs)
+
+
+class DStringFStringInteractionTestCase(unittest.TestCase):
+    def assertAllRaise(self, exception_type, regex, error_strings):
+        for str in error_strings:
+            with self.subTest(str=str):
+                with self.assertRaisesRegex(exception_type, regex):
+                    eval(str)
+
+    def test_fdstring(self):
+        g = {"world": 42}
+
+        source = r"""
+            Hello
+              {world}
+            """
+        self.assertEqual(fd(source, globals=g), "Hello\n  42\n")
+
+        source = r"""
+            Hello
+          {world}
+            """
+        self.assertEqual(fd(source, globals=g), "  Hello\n42\n  ")
+
+        source = r"""
+            Hello {world} Lorum
+            ipsum dolor sit amet,
+            """
+        self.assertEqual(fd(source, globals=g), "Hello 42 Lorum\nipsum dolor sit amet,\n")
+
+    def test_missing_newline_in_f_variants(self):
+        exprs = [
+            'df"""x"""',
+            'drf"""x"""',
+        ]
+        self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs)
+
+    def test_fdstring_conversion_and_format(self):
+        g = {"x": 3.14159, "name": "Alice"}
+
+        source = r"""
+            {x:.2f} {name!r}
+            """
+        self.assertEqual(fd(source, globals=g), "3.14 'Alice'\n")
+
+        source = r"""
+            {x=}
+            """
+        self.assertEqual(fd(source, globals=g), "x=3.14159\n")
+
+    def test_concat_with_fstring(self):
+        expr = 'd"""\n    hello\n    """ f"world"'
+        self.assertEqual(eval(expr), "hello\nworld")
+
+
+class DStringTStringInteractionTestCase(unittest.TestCase, TStringBaseCase):
+    def assertAllRaise(self, exception_type, regex, error_strings):
+        for str in error_strings:
+            with self.subTest(str=str):
+                with self.assertRaisesRegex(exception_type, regex):
+                    eval(str)
+
+    def test_missing_newline_in_t_variants(self):
+        exprs = [
+            'dt"""x"""',
+            'drt"""x"""',
+        ]
+        self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs)
+
+    def test_dtstring_basic(self):
+        name = "Python"
+        t = eval('dt"""\n    Hello, {name}\n    """', {"name": name})
+        self.assertTStringEqual(t, ("Hello, ", "\n"), [(name, "name")])
+
+    def test_drtstring_raw_content(self):
+        t = eval('drt"""\n    keep\\n\n    """')
+        self.assertTStringEqual(t, ("keep\\n\n",), ())
+
+    def test_concat_with_tstring_is_rejected(self):
+        exprs = [
+            'd"""\n    x\n    """ t"hello"',
+            't"hello" d"""\n    x\n    """',
+            'db"""\n    x\n    """ t"hello"',
+            't"hello" db"""\n    x\n    """',
+        ]
+        self.assertAllRaise(
+            SyntaxError,
+            "cannot mix t-string literals with string or bytes literals",
+            exprs,
+        )
+
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_tokenize.py

@@ -3420,7 +3420,7 @@ def determine_valid_prefixes():
         # some uppercase-only prefix is added.
         for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase):
             try:
-                eval(f'{letter}""')
+                eval(f'{letter}"""\n"""')  # d-string needs multiline
                 single_char_valid_prefixes.add(letter.lower())
             except SyntaxError:
                 pass
@@ -3444,7 +3444,7 @@ def determine_valid_prefixes():
                             # because it's a valid expression: not ""
                             continue
                         try:
-                            eval(f'{p}""')
+                            eval(f'{p}"""\n"""')  # d-string needs multiline
 
                             # No syntax error, so p is a valid string
                             # prefix.

Lib/tokenize.py

@@ -86,7 +86,8 @@ def _all_string_prefixes():
     # The valid string prefixes. Only contain the lower case versions,
     #  and don't contain any permutations (include 'fr', but not
     #  'rf'). The various permutations will be generated.
-    _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr']
+    _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'd', 'br', 'fr', 'tr',
+                              'bd', 'rd', 'fd', 'td', 'brd', 'frd', 'trd']
     # if we add binary f-strings, add: ['fb', 'fbr']
     result = {''}
     for prefix in _valid_string_prefixes:

Objects/unicodeobject.c

@@ -13506,8 +13506,8 @@ of all lines in the [src, end).
 It returns the length of the common leading whitespace and sets `output` to
 point to the beginning of the common leading whitespace if length > 0.
 */
-static Py_ssize_t
-search_longest_common_leading_whitespace(
+Py_ssize_t
+_Py_search_longest_common_leading_whitespace(
     const char *const src,
     const char *const end,
     const char **output)
@@ -13602,7 +13602,7 @@ _PyUnicode_Dedent(PyObject *unicode)
     // [whitespace_start, whitespace_start + whitespace_len)
     // describes the current longest common leading whitespace
     const char *whitespace_start = NULL;
-    Py_ssize_t whitespace_len = search_longest_common_leading_whitespace(
+    Py_ssize_t whitespace_len = _Py_search_longest_common_leading_whitespace(
         src, end, &whitespace_start);
 
     if (whitespace_len == 0) {