[torque]: Require formatting for .tq files in presubmit

Bug: v8:7793
Change-Id: I18bd30b36be3c3f89962a2e9c04d8f159c8cfe2b
Reviewed-on: https://chromium-review.googlesource.com/1240415
Commit-Queue: Daniel Clifford <danno@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56202}

PRESUBMIT.py

@@ -74,6 +74,7 @@ def _V8PresubmitChecks(input_api, output_api):
   sys.path.append(input_api.os_path.join(
         input_api.PresubmitLocalPath(), 'tools'))
   from presubmit import CppLintProcessor
+  from presubmit import TorqueFormatProcessor
   from presubmit import SourceProcessor
   from presubmit import StatusFilesProcessor
 
@@ -83,10 +84,18 @@ def _V8PresubmitChecks(input_api, output_api):
       white_list=None,
       black_list=_NO_LINT_PATHS)
 
+  def FilterTorqueFile(affected_file):
+    return input_api.FilterSourceFile(
+      affected_file,
+      white_list=(r'.+\.tq'))
+
   results = []
   if not CppLintProcessor().RunOnFiles(
       input_api.AffectedFiles(file_filter=FilterFile, include_deletes=False)):
     results.append(output_api.PresubmitError("C++ lint check failed"))
+  if not TorqueFormatProcessor().RunOnFiles(
+      input_api.AffectedFiles(file_filter=FilterTorqueFile, include_deletes=False)):
+    results.append(output_api.PresubmitError("Torque format check failed"))
   if not SourceProcessor().RunOnFiles(
       input_api.AffectedFiles(include_deletes=False)):
     results.append(output_api.PresubmitError(

src/builtins/base.tq

@@ -78,7 +78,8 @@ const PACKED_SMI_ELEMENTS:
     constexpr ElementsKind generates 'PACKED_SMI_ELEMENTS';
 const HOLEY_SMI_ELEMENTS:
     constexpr ElementsKind generates 'HOLEY_SMI_ELEMENTS';
-const PACKED_ELEMENTS: constexpr ElementsKind generates 'PACKED_ELEMENTS';
+const PACKED_ELEMENTS:
+    constexpr ElementsKind generates 'PACKED_ELEMENTS';
 const HOLEY_ELEMENTS: constexpr ElementsKind generates 'HOLEY_ELEMENTS';
 const PACKED_DOUBLE_ELEMENTS:
     constexpr ElementsKind generates 'PACKED_DOUBLE_ELEMENTS';
@@ -89,12 +90,16 @@ const DICTIONARY_ELEMENTS:
 
 const UINT8_ELEMENTS: constexpr ElementsKind generates 'UINT8_ELEMENTS';
 const INT8_ELEMENTS: constexpr ElementsKind generates 'INT8_ELEMENTS';
-const UINT16_ELEMENTS: constexpr ElementsKind generates 'UINT16_ELEMENTS';
+const UINT16_ELEMENTS:
+    constexpr ElementsKind generates 'UINT16_ELEMENTS';
 const INT16_ELEMENTS: constexpr ElementsKind generates 'INT16_ELEMENTS';
-const UINT32_ELEMENTS: constexpr ElementsKind generates 'UINT32_ELEMENTS';
+const UINT32_ELEMENTS:
+    constexpr ElementsKind generates 'UINT32_ELEMENTS';
 const INT32_ELEMENTS: constexpr ElementsKind generates 'INT32_ELEMENTS';
-const FLOAT32_ELEMENTS: constexpr ElementsKind generates 'FLOAT32_ELEMENTS';
-const FLOAT64_ELEMENTS: constexpr ElementsKind generates 'FLOAT64_ELEMENTS';
+const FLOAT32_ELEMENTS:
+    constexpr ElementsKind generates 'FLOAT32_ELEMENTS';
+const FLOAT64_ELEMENTS:
+    constexpr ElementsKind generates 'FLOAT64_ELEMENTS';
 const UINT8_CLAMPED_ELEMENTS:
     constexpr ElementsKind generates 'UINT8_CLAMPED_ELEMENTS';
 const BIGUINT64_ELEMENTS:
@@ -138,8 +143,8 @@ const kMaxSafeInteger: constexpr float64 generates 'kMaxSafeInteger';
 const kTruncateMinusZero: constexpr ToIntegerTruncationMode
     generates 'ToIntegerTruncationMode::kTruncateMinusZero';
 
-const kNotTypedArray:
-    constexpr MessageTemplate generates 'MessageTemplate::kNotTypedArray';
+const kNotTypedArray: constexpr MessageTemplate
+    generates 'MessageTemplate::kNotTypedArray';
 const kDetachedOperation: constexpr MessageTemplate
     generates 'MessageTemplate::kDetachedOperation';
 const kBadSortComparisonFunction: constexpr MessageTemplate

tools/presubmit.py

@@ -100,6 +100,27 @@ def CppLintWorker(command):
           ' in your $PATH. Lint check skipped.')
     process.kill()
 
+def TorqueLintWorker(command):
+  try:
+    process = subprocess.Popen(command, stderr=subprocess.PIPE)
+    process.wait()
+    out_lines = ""
+    error_count = 0
+    while True:
+      out_line = process.stderr.readline()
+      if out_line == '' and process.poll() != None:
+        break;
+      out_lines += out_line
+      error_count += 1
+    sys.stdout.write(out_lines)
+    if (error_count != 0):
+        sys.stdout.write("tip: use 'tools/torque/format-torque.py -i <filename>'\n");
+    return error_count
+  except KeyboardInterrupt:
+    process.kill()
+  except:
+    print('Error running format-torque.py')
+    process.kill()
 
 class FileContentsCache(object):
 
@@ -244,7 +265,7 @@ class CppLintProcessor(SourceFileProcessor):
     good_files_cache.Load()
     files = good_files_cache.FilterUnchangedFiles(files)
     if len(files) == 0:
-      print 'No changes in files detected. Skipping cpplint check.'
+      print 'No changes in C/C++ files detected. Skipping cpplint check.'
       return True
 
     filters = ",".join([n for n in LINT_RULES])
@@ -270,10 +291,65 @@ class CppLintProcessor(SourceFileProcessor):
         good_files_cache.RemoveFile(files[i])
 
     total_errors = sum(results)
-    print "Total errors found: %d" % total_errors
+    print "Total C/C++ files found that require formatting: %d" % total_errors
     good_files_cache.Save()
     return total_errors == 0
 
+class TorqueFormatProcessor(SourceFileProcessor):
+  """
+  Check .tq files to verify they follow the Torque style guide.
+  """
+
+  def IsRelevant(self, name):
+    return name.endswith('.tq')
+
+  def GetPathsToSearch(self):
+    dirs = ['third-party', 'src']
+    test_dirs = ['torque']
+    return dirs + [join('test', dir) for dir in test_dirs]
+
+  def GetTorquelintScript(self):
+    torque_tools = os.path.join(TOOLS_PATH, "torque")
+    torque_link = os.path.join(torque_tools, "format-torque.py")
+
+    if os.path.isfile(torque_link):
+      return torque_link
+
+    return None
+
+  def ProcessFiles(self, files):
+    good_files_cache = FileContentsCache('.torquelint-cache')
+    good_files_cache.Load()
+    files = good_files_cache.FilterUnchangedFiles(files)
+    if len(files) == 0:
+      print 'No changes in Torque files detected. Skipping Torque lint check.'
+      return True
+
+    torquelint = self.GetTorquelintScript()
+    if torquelint is None:
+      print('Could not find format-torque. Make sure '
+            'depot_tools is installed and in the path.')
+      sys.exit(1)
+
+    command = [sys.executable, torquelint, '-l']
+
+    commands = [command + [file] for file in files]
+    count = multiprocessing.cpu_count()
+    pool = multiprocessing.Pool(count)
+    try:
+      results = pool.map_async(TorqueLintWorker, commands).get(999999)
+    except KeyboardInterrupt:
+      print "\nCaught KeyboardInterrupt, terminating workers."
+      sys.exit(1)
+
+    for i in range(len(files)):
+      if results[i] > 0:
+        good_files_cache.RemoveFile(files[i])
+
+    total_errors = sum(results)
+    print "Total Torque files requiring formatting: %d" % total_errors
+    good_files_cache.Save()
+    return total_errors == 0
 
 COPYRIGHT_HEADER_PATTERN = re.compile(
     r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
@@ -584,6 +660,8 @@ def Main():
   if not options.no_lint:
     print "Running C++ lint check..."
     success &= CppLintProcessor().RunOnPath(workspace)
+  print "Running Torque formatting check..."
+  success &= TorqueFormatProcessor().RunOnPath(workspace)
   print "Running copyright header, trailing whitespaces and " \
         "two empty lines between declarations check..."
   success &= SourceProcessor().RunOnPath(workspace)

tools/torque/format-torque.py

@@ -22,7 +22,7 @@ def preprocess(input):
   input = re.sub(r'(\s+)case\s*\(([^\:]+)\)(\s*)\:',
       r'\1case \2: /*_TSX*/', input)
   input = re.sub(r'\sgenerates\s+\'([^\']+)\'\s*',
-      r' _GeNeRaT_/*\1@*/', input)
+      r' _GeNeRaTeS00_/*\1@*/', input)
   input = re.sub(r'\sconstexpr\s+\'([^\']+)\'\s*',
       r' _CoNsExP_/*\1@*/', input)
   return input
@@ -37,9 +37,9 @@ def postprocess(output):
       r'case (\1):', output)
   output = re.sub(r'case ([^\:]+)\:\s*\/\*_TSV([^\:]+)\:\*\/',
       r'case (\2: \1):', output)
-  output = re.sub(r'(\n\s*)_GeNeRaT_\s*\/\*([^@]+)@\*\/',
-      r"\1    generates '\2'", output)
-  output = re.sub(r'_GeNeRaT_\s*\/\*([^@]+)@\*\/',
+  output = re.sub(r'\n_GeNeRaTeS00_\s*\/\*([^@]+)@\*\/',
+      r"\n    generates '\1'", output)
+  output = re.sub(r'_GeNeRaTeS00_\s*\/\*([^@]+)@\*\/',
       r"generates '\1'", output)
   output = re.sub(r'_CoNsExP_\s*\/\*([^@]+)@\*\/',
       r"constexpr '\1'", output)
@@ -52,20 +52,28 @@ if len(sys.argv) < 2 or len(sys.argv) > 3:
   sys.exit(-1)
 
 use_stdout = True
-if len(sys.argv) == 3 and sys.argv[1] == '-i':
+lint = False
+if len(sys.argv) == 3:
+  if sys.argv[1] == '-i':
     use_stdout = False
+  if sys.argv[1] == '-l':
+    lint = True
 
 filename = sys.argv[len(sys.argv) - 1]
 
 with open(filename, 'r') as content_file:
   content = content_file.read()
+original_input = content
 p = Popen(['clang-format', '-assume-filename=.ts'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
 output, err = p.communicate(preprocess(content))
 output = postprocess(output)
 rc = p.returncode
 if (rc <> 0):
   sys.exit(rc);
-if use_stdout:
+if lint:
+  if (output != original_input):
+    print >>sys.stderr, filename + ' requires formatting'
+elif use_stdout:
   print output
 else:
   output_file = open(filename, 'w')