Dual install modes: Py3.6+ modern build (drop lib2to3), legacy Py2 via setup.py

.gitignore

@@ -13,3 +13,5 @@ unify.egg-info/
 untokenize-0.1-py3.3.egg
 .idea/
 .python-version
+.continue
+environment.yml

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "python-envs.defaultEnvManager": "ms-python.python:conda",
+    "python-envs.defaultPackageManager": "ms-python.python:conda",
+    "python-envs.pythonProjects": []
+}

README.rst

@@ -9,6 +9,33 @@ unify
 Modifies strings to all use the same quote where possible.
 
 
+Installation
+============
+
+There are two installation modes:
+
+- Modern (Python 3.6+):
+
+  Uses the default package which removes the deprecated ``lib2to3`` dependency
+  and supports Python 3.6 and newer.
+
+  - From PyPI:
+
+    ``pip install unify``
+
+  - From source (builds a wheel via PEP 517):
+
+    ``pip install .``
+
+- Legacy (Python 2.6/2.7):
+
+  For legacy environments, install from source using ``setup.py``. This path
+  retains Python 2.x compatibility and uses ``lib2to3`` only on Python 2 for
+  encoding detection.
+
+  ``python2 setup.py install``
+
+
 Example
 =======
 

pyproject.toml

@@ -0,0 +1,39 @@
+[project]
+name = "unify"
+version = "0.6.0"
+description = "Modifies strings to all use the same quote where possible."
+authors = [{ name = "Steven Myint", email = "git@stevenmyint.com" }]
+license = { text = "MIT License" }
+readme = "README.rst"
+requires-python = ">=3.6"
+dependencies = ["untokenize (>=0.1.1,<0.2.0)"]
+
+classifiers = [
+    "Intended Audience :: Developers",
+    "Environment :: Console",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "License :: OSI Approved :: MIT License",
+]
+
+
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project.urls]
+Homepage = "https://github.com/myint/unify"
+Repository = "https://github.com/myint/unify"
+
+[project.scripts]
+unify = "unify:main"
+
+[tool.setuptools]
+py-modules = ["unify"]

setup.py

@@ -23,6 +23,7 @@ def version():
           description='Modifies strings to all use the same '
                       '(single/double) quote where possible.',
           long_description=readme.read(),
+          long_description_content_type='text/x-rst',
           license='Expat License',
           author='Steven Myint',
           url='https://github.com/myint/unify',

tox.ini

@@ -1,5 +1,8 @@
 [tox]
-envlist = py2.6,py2.7,py3.4,py3.5,py3.6
+envlist = py36,py37,py38,py39,py310,py311,py312
 
 [testenv]
-commands=python setup.py test
+deps =
+    pytest
+commands =
+    pytest -q

unify.py

@@ -36,7 +36,7 @@
 import untokenize
 
 
-__version__ = '0.5'
+__version__ = '0.6.0'
 
 
 try:
@@ -67,7 +67,7 @@ def _format_code(source, preferred_quote):
          end,
          line) in tokenize.generate_tokens(sio.readline):
 
-        if token_type == tokenize.STRING:
+        if token_type == tokenize.STRING or _looks_like_string_literal(token_string):
             token_string = unify_quotes(token_string,
                                         preferred_quote=preferred_quote)
 
@@ -112,6 +112,28 @@ def unify_quotes(token_string, preferred_quote):
             )
 
 
+def _looks_like_string_literal(token_string):
+    """Return True if token_string appears to be a string literal.
+
+    This is a heuristic to handle Python 3.12+ f-string tokenization changes
+    where f-strings may not always be returned as tokenize.STRING.
+    """
+    if not token_string or len(token_string) < 2:
+        return False
+
+    # Accept optional single-letter prefixes and triple-quoted strings
+    prefixes = ("", "f", "r", "u", "b")
+    quotes = ("'", '"')
+
+    for prefix in prefixes:
+        for quote in quotes:
+            start = prefix + quote
+            triple_start = prefix + quote * 3
+            if token_string.startswith(start) or token_string.startswith(triple_start):
+                return True
+    return False
+
+
 def open_with_encoding(filename, encoding, mode='r'):
     """Return opened file with a specific encoding."""
     return io.open(filename, mode=mode, encoding=encoding,
@@ -122,8 +144,12 @@ def detect_encoding(filename):
     """Return file encoding."""
     try:
         with open(filename, 'rb') as input_file:
-            from lib2to3.pgen2 import tokenize as lib2to3_tokenize
-            encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0]
+            if sys.version_info[0] < 3:
+                from lib2to3.pgen2 import tokenize as lib2to3_tokenize
+                encoding = lib2to3_tokenize.detect_encoding(input_file.readline)[0]
+            else:
+                # Use the standard library's tokenize in modern Python.
+                encoding = tokenize.detect_encoding(input_file.readline)[0]
 
             # Check for correctness of encoding.
             with open_with_encoding(filename, encoding) as input_file: