Merge pull request #32970 from compnerd/force-encoding

Use UTF-8 I/O encoding for Python 3

Author: compnerd

utils/gyb.py

@@ -4,6 +4,7 @@
 
 from __future__ import print_function
 
+import io
 import os
 import re
 import sys
@@ -13,7 +14,7 @@
 
 
 try:
-    from cStringIO import StringIO
+    from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
@@ -401,7 +402,7 @@ def __init__(self, filename, template=None):
         if sys.platform == 'win32':
             self.filename = self.filename.replace('\\', '/')
         if template is None:
-            with open(filename) as f:
+            with io.open(filename, encoding='utf-8') as f:
                 self.template = f.read()
         else:
             self.template = template
@@ -736,8 +737,10 @@ def execute(self, context):
             result_string = None
             if isinstance(result, Number) and not isinstance(result, Integral):
                 result_string = repr(result)
-            else:
+            elif isinstance(result, Integral):
                 result_string = str(result)
+            else:
+                result_string = StringIO(result).read()
             context.append_text(
                 result_string, self.filename, self.start_line_number)
 
@@ -803,7 +806,7 @@ def expand(filename, line_directive=_default_line_directive, **local_bindings):
     >>> f.close()
     >>> os.remove(f.name)
     """
-    with open(filename) as f:
+    with io.open(filename, encoding='utf-8') as f:
         t = parse_template(filename, f.read())
         d = os.getcwd()
         os.chdir(os.path.dirname(os.path.abspath(filename)))
@@ -1208,12 +1211,12 @@ def succ(a):
         help='''Bindings to be set in the template's execution context''')
 
     parser.add_argument(
-        'file', type=argparse.FileType(),
+        'file', type=str,
         help='Path to GYB template file (defaults to stdin)', nargs='?',
-        default=sys.stdin)
+        default='-')
     parser.add_argument(
-        '-o', dest='target', type=argparse.FileType('w'),
-        help='Output file (defaults to stdout)', default=sys.stdout)
+        '-o', dest='target', type=str,
+        help='Output file (defaults to stdout)', default='-')
     parser.add_argument(
         '--test', action='store_true',
         default=False, help='Run a self-test')
@@ -1245,15 +1248,23 @@ def succ(a):
             sys.exit(1)
 
     bindings = dict(x.split('=', 1) for x in args.defines)
-    ast = parse_template(args.file.name, args.file.read())
+    if args.file == '-':
+        ast = parse_template('stdin', sys.stdin.read())
+    else:
+        with io.open(args.file, 'r', encoding='utf-8') as f:
+            ast = parse_template(args.file, f.read())
     if args.dump:
         print(ast)
     # Allow the template to open files and import .py files relative to its own
     # directory
-    os.chdir(os.path.dirname(os.path.abspath(args.file.name)))
+    os.chdir(os.path.dirname(os.path.abspath(args.file)))
     sys.path = ['.'] + sys.path
 
-    args.target.write(execute_template(ast, args.line_directive, **bindings))
+    if args.target == '-':
+        sys.stdout.write(execute_template(ast, args.line_directive, **bindings))
+    else:
+        with io.open(args.target, 'w', encoding='utf-8', newline='\n') as f:
+            f.write(execute_template(ast, args.line_directive, **bindings))
 
 
 if __name__ == '__main__':

utils/line-directive

@@ -13,6 +13,7 @@
 from __future__ import print_function
 
 import bisect
+import io
 import os
 import re
 import shlex
@@ -77,7 +78,7 @@ def _make_line_map(target_filename, stream=None):
     [(0, 'box', 1), (1, 'foo.bar', 3), (5, 'baz.txt', 20)]
     """
     result = [(0, target_filename, 1)]
-    input = stream or open(target_filename)
+    input = stream or io.open(target_filename, encoding='utf-8')
     for i, l in enumerate(input.readlines()):
         m = line_pattern.match(l)
         if m:
@@ -142,7 +143,7 @@ def map_line_to_source_file(target_filename, target_line_num):
     map = fline_map(target_filename)
     index = bisect.bisect_left(map, (target_line_num, '', 0))
     base = map[index - 1]
-    return base[1], base[2] + (target_line_num - base[0] - 1)
+    return str(base[1]), base[2] + (target_line_num - base[0] - 1)
 
 
 def map_line_from_source_file(source_filename, source_line_num,