Merge pull request #95 from KennyMcSimpson/fix/cli-utf8-input-files

Read CLI JSON and pointer files as UTF-8

Author: stefankoegl

bin/jsonpointer

@@ -13,7 +13,7 @@ parser = argparse.ArgumentParser(
 # Accept pointer as option, as file or as first positional argument
 ptr_group = parser.add_mutually_exclusive_group()
 
-ptr_group.add_argument('-f', '--pointer-file', type=argparse.FileType('r'),
+ptr_group.add_argument('-f', '--pointer-file', type=argparse.FileType('r', encoding='utf-8'),
                        help='File containing a JSON pointer expression')
 
 ptr_group.add_argument('-p', '--pointer', type=str,
@@ -88,7 +88,7 @@ def resolve_files():
 
     for name in files:
         try:
-            f = argparse.FileType('r')(name)
+            f = argparse.FileType('r', encoding='utf-8')(name)
         except argparse.ArgumentTypeError as e:
             parser.error(str(e))
 

doc/commandline.rst

@@ -31,6 +31,8 @@ The program has the following usage ::
 The pointer can be passed as the first positional argument, with ``-p``, or
 read from a file with ``-f``.
 
+JSON input files and pointer files are read as UTF-8.
+
 The exit status is 0 if the pointer could be resolved on all files, 1 if it
 could not be resolved on at least one file (the remaining files are still
 processed) and 2 on usage errors, such as an invalid pointer or a missing

tests.py

@@ -2,6 +2,7 @@
 
 import copy
 import doctest
+import json
 import os
 import subprocess
 import sys
@@ -566,14 +567,33 @@ def setUp(self):
         self._write('ptr.txt', '/a\n')
 
     def _write(self, name, content):
-        with open(os.path.join(self.dir, name), 'w') as f:
+        with open(os.path.join(self.dir, name), 'w', encoding='utf-8') as f:
             f.write(content)
 
-    def _run(self, *args):
+    def _run(self, *args, utf8=True):
         env = dict(os.environ, PYTHONPATH=self.ROOT, PYTHONUTF8='1')
+        if not utf8:
+            # Keep pipes in UTF-8 while exercising locale-dependent file I/O.
+            env.update(PYTHONUTF8='0', PYTHONIOENCODING='utf-8',
+                       LC_ALL='C', PYTHONCOERCECLOCALE='0')
         return subprocess.run([sys.executable, self.SCRIPT] + list(args),
                               cwd=self.dir, env=env, capture_output=True,
-                              text=True)
+                              encoding='utf-8')
+
+    def test_utf8_json_file_without_utf8_mode(self):
+        self._write('unicode.json', '{"value": "🚀"}')
+        proc = self._run('/value', 'unicode.json', utf8=False)
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertEqual(json.loads(proc.stdout), '🚀')
+        self.assertEqual(proc.stderr, '')
+
+    def test_utf8_pointer_file_without_utf8_mode(self):
+        self._write('unicode.json', json.dumps({'ä': 'expected'}))
+        self._write('unicode.txt', '/ä\n')
+        proc = self._run('-f', 'unicode.txt', 'unicode.json', utf8=False)
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertEqual(json.loads(proc.stdout), 'expected')
+        self.assertEqual(proc.stderr, '')
 
     def test_positional_pointer(self):
         proc = self._run('/a', 'a.json', 'b.json')