[d8] Allow reading files from a TCP socket

Add a --read-from-tcp flag to d8, which makes file reads (including
reading files from arguments, and the load and read builtins) read the
file contents off a TCP socket using a simple request/response protocol.

On top of this, add a script for transparently running d8 on an android
device using adb. The script loads d8 onto the device, starts a file
server providing the above protocol, and uses the above flag to run a d8
which loads javascript sources off the computer rather than off the
device.

Change-Id: Icaa0577beb9bcd4f93476faa3ad8fb8b0a165e6e
Reviewed-on: https://chromium-review.googlesource.com/623790
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#47511}

src/d8-posix.cc

@@ -4,10 +4,12 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <netinet/ip.h>
 #include <signal.h>
 #include <stdlib.h>
 #include <string.h>
 #include <sys/select.h>
+#include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/time.h>
 #include <sys/types.h>
@@ -747,6 +749,107 @@ void Shell::UnsetEnvironment(const v8::FunctionCallbackInfo<v8::Value>& args) {
   unsetenv(*var);
 }
 
+char* Shell::ReadCharsFromTcpPort(const char* name, int* size_out) {
+  DCHECK_GE(Shell::options.read_from_tcp_port, 0);
+
+  int sockfd = socket(PF_INET, SOCK_STREAM, 0);
+  if (sockfd < 0) {
+    i::PrintF(stderr, "Failed to create IPv4 socket\n");
+    return nullptr;
+  }
+
+  // Create an address for localhost:PORT where PORT is specified by the shell
+  // option --read-from-tcp-port.
+  sockaddr_in serv_addr;
+  memset(&serv_addr, 0, sizeof(sockaddr_in));
+  serv_addr.sin_family = AF_INET;
+  serv_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+  serv_addr.sin_port = htons(Shell::options.read_from_tcp_port);
+
+  if (connect(sockfd, reinterpret_cast<sockaddr*>(&serv_addr),
+              sizeof(serv_addr)) < 0) {
+    i::PrintF(stderr, "Failed to connect to localhost:%d\n",
+              Shell::options.read_from_tcp_port);
+    close(sockfd);
+    return nullptr;
+  }
+
+  // The file server follows the simple protocol for requesting and receiving
+  // a file with a given filename:
+  //
+  //   REQUEST client -> server: {filename}"\0"
+  //   RESPONSE server -> client: {4-byte file-length}{file contents}
+  //
+  // i.e. the request sends the filename with a null terminator, and response
+  // sends the file contents by sending the length (as a 4-byte big-endian
+  // value) and the contents.
+
+  // If the file length is <0, there was an error sending the file, and the
+  // rest of the response is undefined (and may, in the future, contain an error
+  // message). The socket should be closed to avoid trying to interpret the
+  // undefined data.
+
+  // REQUEST
+  // Send the filename.
+  size_t sent_len = 0;
+  size_t name_len = strlen(name) + 1;  // Includes the null terminator
+  while (sent_len < name_len) {
+    ssize_t sent_now = send(sockfd, name + sent_len, name_len - sent_len, 0);
+    if (sent_now < 0) {
+      i::PrintF(stderr, "Failed to send %s to localhost:%d\n", name,
+                Shell::options.read_from_tcp_port);
+      close(sockfd);
+      return nullptr;
+    }
+    sent_len += sent_now;
+  }
+
+  // RESPONSE
+  // Receive the file.
+  ssize_t received = 0;
+
+  // First, read the (zero-terminated) file length.
+  uint32_t big_endian_file_length;
+  received = recv(sockfd, &big_endian_file_length, 4, 0);
+  // We need those 4 bytes to read off the file length.
+  if (received < 4) {
+    i::PrintF(stderr, "Failed to receive %s's length from localhost:%d\n", name,
+              Shell::options.read_from_tcp_port);
+    close(sockfd);
+    return nullptr;
+  }
+  // Reinterpretet the received file length as a signed big-endian integer.
+  int32_t file_length = bit_cast<int32_t>(htonl(big_endian_file_length));
+
+  if (file_length < 0) {
+    i::PrintF(stderr, "Received length %d for %s from localhost:%d\n",
+              file_length, name, Shell::options.read_from_tcp_port);
+    close(sockfd);
+    return NULL;
+  }
+
+  // Allocate the output array.
+  char* chars = new char[file_length];
+
+  // Now keep receiving and copying until the whole file is received.
+  ssize_t total_received = 0;
+  while (total_received < file_length) {
+    received =
+        recv(sockfd, chars + total_received, file_length - total_received, 0);
+    if (received < 0) {
+      i::PrintF(stderr, "Failed to receive %s from localhost:%d\n", name,
+                Shell::options.read_from_tcp_port);
+      close(sockfd);
+      delete[] chars;
+      return NULL;
+    }
+    total_received += received;
+  }
+
+  close(sockfd);
+  *size_out = file_length;
+  return chars;
+}
 
 void Shell::AddOSMethods(Isolate* isolate, Local<ObjectTemplate> os_templ) {
   if (options.enable_os_system) {

src/d8-windows.cc

@@ -10,6 +10,11 @@ namespace v8 {
 
 void Shell::AddOSMethods(Isolate* isolate, Local<ObjectTemplate> os_templ) {}
 
+char* Shell::ReadCharsFromTcpPort(const char* name, int* size_out) {
+  // TODO(leszeks): No reason this shouldn't exist on windows.
+  return nullptr;
+}
+
 void Shell::Exit(int exit_code) {
   // Use TerminateProcess avoid races between isolate threads and
   // static destructors.

src/d8.cc

@@ -2048,6 +2048,10 @@ static FILE* FOpen(const char* path, const char* mode) {
 }
 
 static char* ReadChars(const char* name, int* size_out) {
+  if (Shell::options.read_from_tcp_port >= 0) {
+    return Shell::ReadCharsFromTcpPort(name, size_out);
+  }
+
   FILE* file = FOpen(name, "rb");
   if (file == NULL) return NULL;
 
@@ -2717,6 +2721,11 @@ bool Shell::SetOptions(int argc, char* argv[]) {
     } else if (strcmp(argv[i], "--disable-in-process-stack-traces") == 0) {
       options.disable_in_process_stack_traces = true;
       argv[i] = NULL;
+#ifdef V8_OS_POSIX
+    } else if (strncmp(argv[i], "--read-from-tcp-port=", 21) == 0) {
+      options.read_from_tcp_port = atoi(argv[i] + 21);
+      argv[i] = NULL;
+#endif  // V8_OS_POSIX
     } else if (strcmp(argv[i], "--enable-os-system") == 0) {
       options.enable_os_system = true;
       argv[i] = NULL;

src/d8.h

@@ -305,7 +305,8 @@ class ShellOptions {
         trace_enabled(false),
         trace_config(NULL),
         lcov_file(NULL),
-        disable_in_process_stack_traces(false) {}
+        disable_in_process_stack_traces(false),
+        read_from_tcp_port(-1) {}
 
   ~ShellOptions() {
     delete[] isolate_sources;
@@ -337,6 +338,7 @@ class ShellOptions {
   const char* trace_config;
   const char* lcov_file;
   bool disable_in_process_stack_traces;
+  int read_from_tcp_port;
   bool enable_os_system = false;
 };
 
@@ -462,6 +464,8 @@ class Shell : public i::AllStatic {
   static void SetWaitUntilDone(Isolate* isolate, bool value);
   static bool IsWaitUntilDone(Isolate* isolate);
 
+  static char* ReadCharsFromTcpPort(const char* name, int* size_out);
+
  private:
   static Global<Context> evaluation_context_;
   static base::OnceType quit_once_;

tools/adb-d8.py

+#!/usr/bin/env python
+
+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Runs an android build of d8 over adb, with any given arguments. Files
+# requested by d8 are transferred on-demand from the caller, by reverse port
+# forwarding a simple TCP file server from the computer to the android device.
+#
+# Usage:
+#    adb-d8.py [-v|--verbose] <build_dir> [<d8_args>...]
+#
+# Options:
+#    -v|--verbose   Print verbose information.
+#    <build_dir>    The directory containing the android build of d8.
+#    <d8_args>...   The arguments passed through to d8.
+
+from __future__ import print_function
+
+import os
+import sys
+import struct
+import threading
+import subprocess
+import SocketServer # TODO(leszeks): python 3 compatibility
+
+def CreateFileHandlerClass(root_path, verbose):
+  class FileHandler(SocketServer.BaseRequestHandler):
+    def handle(self):
+      data = self.request.recv(1024);
+      while data[-1] != "\0":
+        data += self.request.recv(1024);
+
+      filename = data[0:-1]
+
+      try:
+        filename = os.path.abspath(filename)
+
+        if not filename.startswith(root_path):
+          raise Exception("{} not in root {}".format(filename, root_path))
+        if not os.path.isfile(filename):
+          raise Exception("{} is not a file".format(filename))
+
+        if verbose:
+          sys.stdout.write("Serving {}\r\n".format(os.path.relpath(filename)))
+
+        with open(filename) as f:
+          contents = f.read();
+          self.request.sendall(struct.pack("!i", len(contents)))
+          self.request.sendall(contents)
+
+      except Exception as e:
+        if verbose:
+          sys.stderr.write(
+            "Request failed ({})\n".format(e).replace('\n','\r\n'))
+        self.request.sendall(struct.pack("!i", -1))
+
+  return FileHandler
+
+
+def TransferD8ToDevice(adb, build_dir, device_d8_dir, verbose):
+  files_to_copy = ["d8", "natives_blob.bin", "snapshot_blob.bin"]
+
+  # Pipe the output of md5sum from the local computer to the device, checking
+  # the md5 hashes on the device.
+  local_md5_sum_proc = subprocess.Popen(
+    ["md5sum"] + files_to_copy,
+    cwd=build_dir,
+    stdout=subprocess.PIPE
+  )
+  device_md5_check_proc = subprocess.Popen(
+    [
+      adb, "shell",
+      "mkdir -p '{0}' ; cd '{0}' ; md5sum -c -".format(device_d8_dir)
+    ],
+    stdin=local_md5_sum_proc.stdout,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE
+  )
+
+  # Push any files which failed the md5 check.
+  (stdoutdata, stderrdata) = device_md5_check_proc.communicate()
+  for line in stdoutdata.split('\n'):
+    if line.endswith(": FAILED"):
+      filename = line[:-len(": FAILED")]
+      if verbose:
+        print("Updating {}...".format(filename))
+      subprocess.check_call([
+        adb, "push",
+        os.path.join(build_dir, filename),
+        device_d8_dir
+      ], stdout=sys.stdout if verbose else open(os.devnull, 'wb'))
+
+
+def AdbForwardDeviceToLocal(adb, device_port, server_port, verbose):
+  if verbose:
+    print("Forwarding device:{} to localhost:{}...".format(
+      device_port, server_port))
+
+  subprocess.check_call([
+    adb, "reverse",
+    "tcp:{}".format(device_port),
+    "tcp:{}".format(server_port)
+  ])
+
+
+def AdbRunD8(adb, device_d8_dir, device_port, d8_args, verbose):
+  # Single-quote the arguments to d8, and concatenate them into a string.
+  d8_arg_str = " ".join("'{}'".format(a) for a in d8_args)
+  d8_arg_str = "--read-from-tcp-port='{}' ".format(device_port) + d8_arg_str
+
+  # Don't use os.path.join for d8 because we care about the device's os, not
+  # the host os.
+  d8_str = "{}/d8 {}".format(device_d8_dir, d8_arg_str)
+
+  if verbose:
+    print("Running adb shell -t \"{}\"".format(d8_str))
+
+  # Run adb shell with -t to have a tty if we run d8 without a script.
+  return subprocess.call([adb, "shell", "-t", d8_str])
+
+
+def PrintUsage(file=sys.stdout):
+  print("Usage: adb-d8.py [-v|--verbose] [--] <build_dir> [<d8 args>...]",
+    file=file)
+
+
+def PrintHelp(file=sys.stdout):
+  print("""Usage:
+   adb-d8.py [-v|--verbose] [--] <build_dir> [<d8_args>...]
+   adb-d8.py -h|--help
+
+Options:
+   -h|--help      Show this help message and exit
+   -v|--verbose   Print verbose output
+   <build_dir>    The directory containing the android build of d8
+   <d8_args>...   The arguments passed through to d8""", file=file)
+
+
+def Main():
+  if len(sys.argv) < 2:
+    PrintUsage(sys.stderr)
+    return 1
+
+  script_dir = os.path.dirname(sys.argv[0])
+  # Use the platform-tools version of adb so that we know it has the reverse
+  # command.
+  adb = os.path.join(
+    script_dir,
+    "../third_party/android_tools/sdk/platform-tools/adb"
+  )
+
+  # Read off any command line flags before build_dir (or --). Do this
+  # manually, rather than using something like argparse, to be able to split
+  # the adb-d8 options from the passthrough d8 options.
+  verbose = False
+  for arg_index,arg in enumerate(sys.argv[1:], start=1):
+    if not arg.startswith("-"):
+      break
+    elif arg == "--":
+      arg_index += 1
+      break
+    elif arg == "-h" or arg == "--help":
+      PrintHelp(sys.stdout)
+      return 0
+    elif arg == "-v" or arg == "--verbose":
+      verbose = True
+    else:
+      print("ERROR: Unrecognised option: {}".format(arg))
+      PrintUsage(sys.stderr)
+      return 1
+
+  # Transfer d8 (and dependencies) to the device.
+  build_dir = os.path.abspath(sys.argv[arg_index])
+  device_d8_dir = '/data/local/tmp/v8'
+
+  TransferD8ToDevice(adb, build_dir, device_d8_dir, verbose)
+
+  # Start a file server for the files d8 might need.
+  script_root_dir = os.path.abspath(os.curdir)
+  server = SocketServer.TCPServer(
+    ("localhost", 0), # 0 means an arbitrary unused port.
+    CreateFileHandlerClass(script_root_dir, verbose)
+  )
+
+  try:
+    # Start the file server in its own thread.
+    server_thread = threading.Thread(target=server.serve_forever)
+    server_thread.daemon = True
+    server_thread.start()
+
+    # Port-forward the given device port to the file server.
+    # TODO(leszeks): Pick an unused device port.
+    # TODO(leszeks): Remove the port forwarding on exit.
+    server_ip, server_port = server.server_address
+    device_port = 4444
+    AdbForwardDeviceToLocal(adb, device_port, server_port, verbose)
+
+    # Run d8 over adb with the remaining arguments, using the given device
+    # port to forward file reads.
+    return AdbRunD8(
+      adb, device_d8_dir, device_port, sys.argv[arg_index+1:], verbose)
+
+  finally:
+    if verbose:
+      print("Shutting down file server...")
+    server.shutdown()
+    server.server_close()
+
+if __name__ == '__main__':
+  sys.exit(Main())