setup_color: Use native ANSI support on Windows 10

Windows 10 has support for native ANSI control codes.
This means we don't have to wrap stdout/stderr with Colorama, which uses old CMD APIs to handle these sequences.

This change inadvertently also works around an issue on Python 3 where Colorama errors when bytes are output by gclient_utils.CheckCallAndFilter due to not wrapping sys.stdout.buffer.

Bug: 942522
Change-Id: Ic7dde620b01bd94eb50de353bfc0276cad045a45
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1735566
Commit-Queue: Raul Tambre <raul@tambre.ee>
Reviewed-by: Marc-Antoine Ruel <maruel@chromium.org>
Auto-Submit: Raul Tambre <raul@tambre.ee>

setup_color.py

@@ -5,13 +5,40 @@
 
 from __future__ import print_function
 
+import ctypes
 import os
+import platform
+import subprocess
 import sys
 from third_party import colorama
 
 IS_TTY = None
 OUT_TYPE = 'unknown'
 
+
+def enable_native_ansi():
+  """Enables native ANSI sequences in console. Windows 10 only.
+
+  Returns whether successful.
+  """
+  kernel32 = ctypes.windll.kernel32
+  ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04
+
+  out_handle = kernel32.GetStdHandle(subprocess.STD_OUTPUT_HANDLE)
+  mode = ctypes.wintypes.DWORD()
+  if kernel32.GetConsoleMode(out_handle, ctypes.byref(mode)) == 0:
+    print('kernel32.GetConsoleMode failed')
+    return False
+
+  if not (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING):
+    if kernel32.SetConsoleMode(
+        out_handle, mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0:
+      print('kernel32.SetConsoleMode to enable ANSI sequences failed')
+      return False
+
+  return True
+
+
 def init():
   # should_wrap instructs colorama to wrap stdout/stderr with an ASNI colorcode
   # interpreter that converts them to SetConsoleTextAttribute calls. This only
@@ -30,7 +57,6 @@ def init():
     OUT_TYPE = 'file (win)'
 
     import msvcrt
-    import ctypes
     h = msvcrt.get_osfhandle(sys.stdout.fileno())
     # h is the win32 HANDLE for stdout.
     ftype = ctypes.windll.kernel32.GetFileType(h)
@@ -84,6 +110,10 @@ def init():
     else:
       # A normal file, or an unknown file type.
       pass
+
+    # Enable native ANSI color codes on Windows 10.
+    if IS_TTY and platform.release() == '10':
+      should_wrap = not enable_native_ansi()
   else:
     # This is non-windows, so we trust isatty.
     OUT_TYPE = 'pipe or file'