[tools] Add scripts for compile_commands.json and VSCode

This adds two convenience scripts:
- update-vscode.sh downloads/updates Visual Studio Code
- update-compile-commands.py prepares for code indexers like clangd
  by creating compile_commands.json (for all architectures), updating
  generated sources, and compiling the Torque Language Server.

No-try: true
Change-Id: I64a15dc298f4312a9b296762593234c40f542b06
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2317355
Auto-Submit: Jakob Kummerow <jkummerow@chromium.org>
Reviewed-by: Victor Gomes <victorgomes@chromium.org>
Commit-Queue: Jakob Kummerow <jkummerow@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69105}

tools/dev/update-compile-commands.py

+#!/usr/bin/env python3
+# Copyright 2020 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""\
+Creates a "compile_commands.json" file for V8, for the needs of clangd and
+similar code indexers. Also updates generated C++ sources, and compiles the
+Torque Language Server, for a complete code indexing experience.
+"""
+
+import json
+import os
+import subprocess
+import sys
+
+PYLIB_PATH = 'tools/clang/pylib'
+GM_PATH = 'tools/dev'
+PYLIB_CHECK = os.path.join(PYLIB_PATH, 'clang', 'compile_db.py')
+GM_CHECK = os.path.join(GM_PATH, 'gm.py')
+def CheckRelativeImport(path):
+  if not os.path.exists(path):
+    print("Error: Please run this script from the root of a V8 checkout. %s "
+          "must be a valid relative path." % path)
+    sys.exit(1)
+CheckRelativeImport(PYLIB_CHECK)
+CheckRelativeImport(GM_CHECK)
+
+sys.path.insert(0, PYLIB_PATH)
+from clang import compile_db
+
+sys.path.insert(0, GM_PATH)
+import gm
+
+def _Call(cmd, silent=False):
+  if not silent: print("# %s" % cmd)
+  return subprocess.call(cmd, shell=True)
+
+def _Write(filename, content):
+  with open(filename, "w") as f:
+    f.write(content)
+
+def PrepareBuildDir(arch, mode):
+  build_dir = os.path.join("out", "%s.%s" % (arch, mode))
+  if not os.path.exists(build_dir):
+    print("# mkdir -p %s" % build_dir)
+    os.makedirs(build_dir)
+  args_gn = os.path.join(build_dir, "args.gn")
+  if not os.path.exists(args_gn):
+    conf = gm.Config(arch, mode, [])
+    _Write(args_gn, conf.GetGnArgs())
+  build_ninja = os.path.join(build_dir, "build.ninja")
+  if not os.path.exists(build_ninja):
+    code = _Call("gn gen %s" % build_dir)
+    if code != 0: raise Error("gn gen failed")
+  else:
+    _Call("ninja -C %s build.ninja" % build_dir)
+  return build_dir
+
+def AddTargetsForArch(arch, combined):
+  build_dir = PrepareBuildDir(arch, "debug")
+  commands = compile_db.ProcessCompileDatabaseIfNeeded(
+                compile_db.GenerateWithNinja(build_dir, ["all"]))
+  added = 0
+  for c in commands:
+    key = c["file"]
+    if key not in combined:
+      combined[key] = c
+      added += 1
+  print("%s: added %d compile commands" % (arch, added))
+
+def UpdateCompileCommands():
+  print(">>> Updating compile_commands.json...")
+  combined = {}
+  AddTargetsForArch("x64", combined)
+  AddTargetsForArch("ia32", combined)
+  AddTargetsForArch("arm", combined)
+  AddTargetsForArch("arm64", combined)
+  commands = []
+  for key in combined:
+    commands.append(combined[key])
+  _Write("compile_commands.json", json.dumps(commands, indent=2))
+
+def CompileLanguageServer():
+  print(">>> Compiling Torque Language Server...")
+  PrepareBuildDir("x64", "release")
+  _Call("autoninja -C out/x64.release torque-language-server")
+
+def GenerateCCFiles():
+  print(">>> Generating generated C++ source files...")
+  # This must be called after UpdateCompileCommands().
+  assert os.path.exists("out/x64.debug/build.ninja")
+  _Call("autoninja -C out/x64.debug v8_generated_cc_files")
+
+def StartGoma():
+  gomadir = gm.DetectGoma()
+  if (gomadir is not None and
+      _Call("ps -e | grep compiler_proxy > /dev/null", silent=True) != 0):
+    _Call("%s/goma_ctl.py ensure_start" % gomadir)
+
+if __name__ == "__main__":
+  StartGoma()
+  CompileLanguageServer()
+  UpdateCompileCommands()
+  GenerateCCFiles()

tools/dev/update-vscode.sh

+#!/bin/bash
+# Copyright 2020 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# The purpose of this script is to make it easy to download/update
+# Visual Studio Code on Linux distributions where for whatever reason there
+# is no good way to do so via the package manager.
+
+# Version of this script: 2020.07.04
+
+# Basic checking of arguments: want at least one, and it's not --help.
+VERSION="$1"
+[ -z "$VERSION" -o \
+    "$VERSION" == "-h" -o \
+    "$VERSION" == "--help" -o \
+    "$VERSION" == "help" ] && {
+  echo "Usage: $0 <version>"
+  echo "<version> may be --auto for auto-detecting the latest available."
+  exit 1
+}
+
+die() {
+  echo "Error: $1"
+  exit 1
+}
+
+if [ "$VERSION" == "--auto" -o "$VERSION" == "auto" ]; then
+  echo "Searching online for latest available version..."
+  # Where to find the latest available version (we assume that it's mentioned
+  # in the first 1000 characters, which is true as of 2020-07).
+  AVAILABLE_PACKAGES_URL="https://packages.microsoft.com/repos/vscode/dists/stable/main/binary-amd64/Packages"
+  VERSION=$(curl "$AVAILABLE_PACKAGES_URL" --range 0-1000 --silent \
+            | grep "^Version: " \
+            | sed 's/[^0-9]*\([0-9.]*\).*/\1/')
+  if [ -z "$VERSION" ]; then
+    die "Detecting latest version failed, please specify it manually."
+  else
+    echo "Latest version found: $VERSION"
+  fi
+fi
+
+# Constant definitions for local paths. Edit these to your liking.
+VSCODE_DIR="$HOME/vscode"
+BACKUP_DIR="$HOME/vscode.prev"
+DOWNLOADS_DIR="$HOME/Downloads"
+DOWNLOAD_FILE="$DOWNLOADS_DIR/vscode-$VERSION.tar.gz"
+DESKTOP_FILE_DIR="$HOME/.local/share/applications"
+DESKTOP_FILE="$DESKTOP_FILE_DIR/code.desktop"
+
+# Constant definitions for remote/upstream things. Might need to be updated
+# when upstream changes things.
+# Where to find the version inside VS Code's installation directory.
+PACKAGE_JSON="$VSCODE_DIR/resources/app/package.json"
+ICON="$VSCODE_DIR/resources/app/resources/linux/code.png"
+# Where to download the archive.
+DOWNLOAD_URL="https://update.code.visualstudio.com/$VERSION/linux-x64/stable"
+CODE_BIN="$VSCODE_DIR/bin/code"
+
+# Check for "code" in $PATH; create a symlink if we can find a good place.
+SYMLINK=$(which code)
+if [ -z "$SYMLINK" ]; then
+  IFS=':' read -ra PATH_ARRAY <<< "$PATH"
+  for P in "${PATH_ARRAY[@]}"; do
+    if [ "$P" == "$HOME/bin" -o \
+         "$P" == "$HOME/local/bin" -o \
+         "$P" == "$HOME/.local/bin" ]; then
+      LOCALBIN="$P"
+      break
+    fi
+  done
+  if [ -n "$LOCALBIN" ]; then
+    echo "Adding symlink to $LOCALBIN..."
+    if [ ! -d "$LOCALBIN" ]; then
+      mkdir -p "$LOCALBIN" || die "Failed to create $LOCALBIN."
+    fi
+    ln -s "$CODE_BIN" "$LOCALBIN/code" || die "Failed to create symlink."
+  else
+    echo "Please put a symlink to $CODE_BIN somewhere on your \$PATH."
+  fi
+fi
+
+if [ ! -r "$DESKTOP_FILE" ]; then
+  echo "Creating .desktop file..."
+  mkdir -p "$DESKTOP_FILE_DIR" || die "Failed to create .desktop directory."
+  cat <<EOF > "$DESKTOP_FILE"
+#!/usr/bin/env xdg-open
+[Desktop Entry]
+Name=Visual Studio Code
+Comment=Code Editing. Redefined.
+GenericName=Text Editor
+Exec=$CODE_BIN --unity-launch %F
+Icon=$ICON
+Type=Application
+StartupNotify=false
+StartupWMClass=Code
+Categories=Utility;TextEditor;Development;IDE;
+MimeType=text/plain;inode/directory;
+Actions=new-empty-window;
+Keywords=vscode;
+
+X-Desktop-File-Install-Version=0.24
+
+[Desktop Action new-empty-window]
+Name=New Empty Window
+Exec=$CODE_BIN --new-window %F
+Icon=$ICON
+EOF
+  chmod +x "$DESKTOP_FILE" || die "Failed to make .desktop file executable."
+fi
+
+# Find currently installed version.
+if [ -d "$VSCODE_DIR" ]; then
+  if [ ! -r "$PACKAGE_JSON" ]; then
+    die "$PACKAGE_JSON file not found, this script must be updated."
+  fi
+  INSTALLED=$(grep '"version":' "$PACKAGE_JSON" \
+              | sed 's/[^0-9]*\([0-9.]*\).*/\1/')
+  echo "Detected installed version: $INSTALLED"
+  if [ "$VERSION" == "$INSTALLED" ] ; then
+    echo "You already have that version."
+    exit 0
+  else
+    echo "Updating from $INSTALLED to $VERSION..."
+  fi
+fi
+
+if [ ! -r "$DOWNLOAD_FILE" ]; then
+  echo "Downloading..."
+  if [ ! -d "$DOWNLOADS_DIR" ]; then
+    mkdir -p "$DOWNLOADS_DIR" || die "Failed to create $DOWNLOADS_DIR."
+  fi
+  wget "$DOWNLOAD_URL" -O "$DOWNLOAD_FILE" || die "Downloading failed."
+else
+  echo "$DOWNLOAD_FILE already exists; delete it to force re-download."
+fi
+
+echo "Extracting..."
+TAR_DIR=$(tar -tf "$DOWNLOAD_FILE" | head -1)
+[ -z "$TAR_DIR" ] && die "Couldn't read archive."
+TMP_DIR=$(mktemp -d)
+tar -C "$TMP_DIR" -xf "$DOWNLOAD_FILE" || {
+  rm -rf "$TMP_DIR"
+  die "Extracting failed."
+}
+
+if [ -d "$BACKUP_DIR" ]; then
+  echo "Deleting previous backup..."
+  rm -rf "$BACKUP_DIR"
+fi
+
+if [ -d "$VSCODE_DIR" ]; then
+  echo "Moving previous installation..."
+  mv "$VSCODE_DIR" "$BACKUP_DIR"
+fi
+
+echo "Installing new version..."
+mv "$TMP_DIR/$TAR_DIR" "$VSCODE_DIR"
+rmdir "$TMP_DIR"
+
+echo "All done, enjoy coding!"