Add code to 'fix' python encoding and it's unit test.

With this code in action, I can successfully print arabic and chinese on linux
and cygwin. It fails to print chinese on my Mac but prints arabic. It prints '?'
on Windows console but it *doesn't throw* which is the biggest improvement here.
It was particularly a problem on windows because WindowsError's description text
is in the current ANSI code page so it failed to print if the text was not pure
ASCII, like error message when using Windows' French UI.

R=dpranke@chromium.org
BUG=none
TEST=manual unit test ./tests/fix_encoding_test.py and it shouldn't throw

Review URL: http://codereview.chromium.org/6676090

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@79006 0039d316-1c4b-4281-b951-d872f2087c98

PRESUBMIT.py

@@ -9,6 +9,7 @@ details on the presubmit API built into gcl.
 """
 
 UNIT_TESTS = [
+  'tests.fix_encoding_test',
   'tests.gcl_unittest',
   'tests.gclient_scm_test',
   'tests.gclient_smoketest',

gcl.py

@@ -40,6 +40,7 @@ import breakpad  # pylint: disable=W0611
 # gcl now depends on gclient.
 from scm import SVN
 
+import fix_encoding
 import gclient_utils
 import owners
 import presubmit_support
@@ -1473,4 +1474,5 @@ def main(argv):
 
 
 if __name__ == "__main__":
+  fix_encoding.fix_encoding()
   sys.exit(main(sys.argv[1:]))

gclient.py

@@ -64,6 +64,7 @@ import urllib
 
 import breakpad  # pylint: disable=W0611
 
+import fix_encoding
 import gclient_scm
 import gclient_utils
 from third_party.repo.progress import Progress
@@ -1266,6 +1267,7 @@ def Main(argv):
 
 
 if '__main__' == __name__:
+  fix_encoding.fix_encoding()
   sys.exit(Main(sys.argv[1:]))
 
 # vim: ts=2:sw=2:tw=80:et:

presubmit_support.py

@@ -50,6 +50,7 @@ except ImportError:
     import simplejson as json  # pylint: disable=F0401
 
 # Local imports.
+import fix_encoding
 import gclient_utils
 import owners
 import presubmit_canned_checks
@@ -142,18 +143,18 @@ class OutputApi(object):
     def handle(self, output):
       output.write(self._message)
       output.write('\n')
-      if len(self._items) > 0:
-        output.write('  ' + ' \\\n  '.join(map(str, self._items)) + '\n')
+      for index, item in enumerate(self._items):
+        output.write('  ')
+        # Write separately in case it's unicode.
+        output.write(item)
+        if index < len(self._items) - 1:
+          output.write(' \\')
+        output.write('\n')
       if self._long_text:
-        # Sometimes self._long_text is a ascii string, a codepage string
-        # (on windows), or a unicode object.
-        try:
-          long_text = self._long_text.decode()
-        except UnicodeDecodeError:
-          long_text = self._long_text.decode('ascii', 'replace')
-
-        output.write('\n***************\n%s\n***************\n' %
-            long_text)
+        output.write('\n***************\n')
+        # Write separately in case it's unicode.
+        output.write(self._long_text)
+        output.write('\n***************\n')
       if self.fatal:
         output.fail()
 
@@ -1192,4 +1193,5 @@ def Main(argv):
 
 
 if __name__ == '__main__':
+  fix_encoding.fix_encoding()
   sys.exit(Main(None))

tests/fix_encoding_test.py

+#!/usr/bin/python
+# coding=utf8
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for fix_encoding.py."""
+
+import os
+import sys
+import unittest
+
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT_DIR)
+
+import fix_encoding
+
+
+class FixEncodingTest(unittest.TestCase):
+  # Nice mix of latin, hebrew, arabic and chinese. Doesn't mean anything.
+  text = u'Héllô 偉大 سيد'
+
+  def test_code_page(self):
+    # Make sure printing garbage won't throw.
+    print self.text.encode() + '\xff'
+    print >> sys.stderr, self.text.encode() + '\xff'
+
+  def test_utf8(self):
+    # Make sure printing utf-8 works.
+    print self.text.encode('utf-8')
+    print >> sys.stderr, self.text.encode('utf-8')
+
+  def test_unicode(self):
+    # Make sure printing unicode works.
+    print self.text
+    print >> sys.stderr, self.text
+
+  def test_default_encoding(self):
+    self.assertEquals('utf-8', sys.getdefaultencoding())
+
+  def test_win_console(self):
+    if sys.platform != 'win32':
+      return
+    # This should fail if redirected. Can be checked with:
+    # python fix_encoding_test.py > a
+    self.assertEquals(
+        sys.stdout.__class__, fix_encoding.WinUnicodeConsoleOutput)
+    self.assertEquals(
+        sys.stderr.__class__, fix_encoding.WinUnicodeConsoleOutput)
+    self.assertEquals(sys.stdout.encoding, sys.getdefaultencoding())
+    self.assertEquals(sys.stderr.encoding, sys.getdefaultencoding())
+
+  def test_multiple_calls(self):
+    # Shouldn't do anything.
+    self.assertEquals(False, fix_encoding.fix_encoding())
+
+
+if __name__ == '__main__':
+  assert fix_encoding.fix_encoding()
+  unittest.main()

tests/gcl_unittest.py

@@ -90,7 +90,8 @@ class GclUnittest(GclTestsBase):
         'OptionallyDoPresubmitChecks', 'REPOSITORY_ROOT', 'REVIEWERS_REGEX',
         'RunShell', 'RunShellWithReturnCode', 'SVN',
         'TryChange', 'UnknownFiles', 'Warn',
-        'attrs', 'breakpad', 'defer_attributes', 'gclient_utils', 'getpass',
+        'attrs', 'breakpad', 'defer_attributes', 'fix_encoding',
+        'gclient_utils', 'getpass',
         'json', 'main', 'need_change', 'need_change_and_args', 'no_args',
         'optparse', 'os', 'owners', 'presubmit_support', 'random', 're',
         'string', 'subprocess', 'suggest_reviewers', 'sys', 'tempfile',

tests/presubmit_unittest.py

@@ -141,7 +141,8 @@ class PresubmitUnittest(PresubmitTestsBase):
       'NotImplementedException', 'OutputApi', 'ParseFiles',
       'PresubmitExecuter', 'PresubmitOutput', 'ScanSubDirs',
       'SvnAffectedFile', 'SvnChange', 'cPickle', 'cStringIO',
-      'exceptions', 'fnmatch', 'gclient_utils', 'glob', 'json', 'load_files',
+      'exceptions', 'fix_encoding', 'fnmatch', 'gclient_utils', 'glob', 'json',
+      'load_files',
       'logging', 'marshal', 'normpath', 'optparse', 'os', 'owners', 'pickle',
       'presubmit_canned_checks', 'random', 're', 'scm', 'subprocess',
       'sys', 'tempfile', 'time', 'traceback', 'types', 'unittest', 'urllib2',

tests/trychange_unittest.py

@@ -45,7 +45,8 @@ class TryChangeUnittest(TryChangeTestsBase):
       'EPILOG', 'Escape', 'GIT', 'GuessVCS', 'GetMungedDiff', 'HELP_STRING',
       'InvalidScript', 'NoTryServerAccess', 'PrintSuccess', 'SCM', 'SVN',
       'TryChange', 'USAGE',
-      'breakpad', 'datetime', 'errno', 'gcl', 'gclient_utils', 'getpass',
+      'breakpad', 'datetime', 'errno', 'fix_encoding', 'gcl', 'gclient_utils',
+      'getpass',
       'json', 'logging', 'optparse', 'os', 'posixpath', 're', 'scm', 'shutil',
       'sys', 'tempfile', 'urllib',
     ]

trychange.py

@@ -39,6 +39,7 @@ try:
 except ImportError:
   gcl = None
 
+import fix_encoding
 import gclient_utils
 import scm
 
@@ -769,4 +770,5 @@ def TryChange(argv,
 
 
 if __name__ == "__main__":
+  fix_encoding.fix_encoding()
   sys.exit(TryChange(None, [], False))