[wasm][tests] Add proposal tests to the wasm spec tests

With this CL we add proposal tests to the wasm-spec-tests. For this I
extended the update-wasm-spec-tests.sh script. Additionally to
generating the spec tests it does the following:

For each proposal it identifies those tests that are different to the
spec tests, and then copies those tests also to the wasm-spec-tests
directory.

Additionally I adjusted the test runner of the wasm spec test to
run the proposal tests with the correct flags.

CC=binji@chromium.org
R=clemensh@chromium.org

Bug: v8:7581
Change-Id: Idb7aa3c0a468ddb65b2ef3421def836561579cd9
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1706470Reviewed-by: Clemens Hammacher <clemensh@chromium.org>
Commit-Queue: Andreas Haas <ahaas@chromium.org>
Cr-Commit-Position: refs/heads/master@{#62777}

test/wasm-spec-tests/testcfg.py

@@ -7,6 +7,15 @@ import os
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
+proposal_flags = [{
+                    'name': 'reference-types',
+                    'flags': ['--experimental-wasm-anyref',
+                               '--no-experimental-wasm-bulk-memory']
+                  },
+                  {
+                    'name': 'bulk-memory-operations',
+                    'flags': ['--experimental-wasm-bulk-memory']
+                  }]
 
 class TestLoader(testsuite.JSTestLoader):
   pass
@@ -23,6 +32,12 @@ class TestCase(testcase.D8TestCase):
   def _get_files_params(self):
     return [os.path.join(self.suite.root, self.path + self._get_suffix())]
 
+  def _get_source_flags(self):
+    for proposal in proposal_flags:
+      if os.sep.join(['proposals', proposal['name']]) in self.path:
+        return proposal['flags']
+    return []
+
 
 def GetSuite(*args, **kwargs):
   return TestSuite(*args, **kwargs)

test/wasm-spec-tests/tests.tar.gz.sha1

-e8bdb558198b944ff8a0df43301f1ff4eb3a91fa
\ No newline at end of file
+b02f00e24b28ad76537a10a788a8be966c3577bd
\ No newline at end of file

test/wasm-spec-tests/wasm-spec-tests.status

@@ -10,6 +10,20 @@
   # the bulk-memory proposal. Since we've enabled bulk-memory by default, we
   # need to update to use its testsuite.
   'tests/linking': [FAIL],
+
+  # TODO(ahaas): Needs investigation, I disable the test for now.
+  'tests/conversions': [PASS, ['system == windows and arch == ia32', FAIL]],
+
+  # TODO(ahaas): Incorporate recent changes to the bulk-memory-operations
+  # proposal.
+  'tests/proposals/bulk-memory-operations/elem': [FAIL],
+  'tests/proposals/bulk-memory-operations/memory_copy': [FAIL],
+  'tests/proposals/bulk-memory-operations/table_copy': [FAIL],
+  'tests/proposals/bulk-memory-operations/table_init': [FAIL],
+  'tests/proposals/bulk-memory-operations/memory_init': [FAIL],
+  'tests/proposals/bulk-memory-operations/memory_fill': [FAIL],
+  'tests/proposals/bulk-memory-operations/bulk': [FAIL],
+  'tests/proposals/bulk-memory-operations/data': [FAIL],
 }],  # ALWAYS
 
 ['arch == mipsel or arch == mips64el or arch == mips or arch == mips64', {

tools/wasm/update-wasm-spec-tests.sh

@@ -13,43 +13,101 @@ set -u
 # non-zero status, or zero if no command exited with a non-zero status
 set -o pipefail
 
+log_and_run() {
+  echo ">>" $*
+  if ! $*; then
+    echo "sub-command failed: $*"
+    exit
+  fi
+}
+
+###############################################################################
+# Setup directories.
+###############################################################################
+
 TOOLS_WASM_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 V8_DIR="${TOOLS_WASM_DIR}/../.."
 SPEC_TEST_DIR=${V8_DIR}/test/wasm-spec-tests
+TMP_DIR=${SPEC_TEST_DIR}/tmp
 
-cd ${V8_DIR}
+log_and_run cd ${V8_DIR}
 
-rm -rf ${SPEC_TEST_DIR}/tests
-mkdir ${SPEC_TEST_DIR}/tests
+log_and_run rm -rf ${SPEC_TEST_DIR}/tests
+log_and_run mkdir ${SPEC_TEST_DIR}/tests
 
-rm -rf ${SPEC_TEST_DIR}/tmp
-mkdir ${SPEC_TEST_DIR}/tmp
+log_and_run mkdir ${SPEC_TEST_DIR}/tests/proposals
 
-./tools/dev/gm.py x64.release d8
+log_and_run rm -rf ${TMP_DIR}
+log_and_run mkdir ${TMP_DIR}
 
-cd ${V8_DIR}/test/wasm-js/data/interpreter
+###############################################################################
+# Generate the spec tests.
+###############################################################################
 
+log_and_run cd ${V8_DIR}/test/wasm-js/data/interpreter
 # The next step requires that ocaml is installed. See the README.md in
 # ${V8_DIR}/test/wasm-js/data/interpreter/.
-make clean all
+log_and_run make clean opt
+
+log_and_run cd ${V8_DIR}/test/wasm-js/data/test/core
+log_and_run cp *.wast ${SPEC_TEST_DIR}/tests/
+
+log_and_run ./run.py --wasm ${V8_DIR}/test/wasm-js/data/interpreter/wasm --out ${TMP_DIR}
+log_and_run cp ${TMP_DIR}/*.js ${SPEC_TEST_DIR}/tests/
 
-cd ${V8_DIR}/test/wasm-js/data/test/core
+###############################################################################
+# Generate the proposal tests.
+###############################################################################
 
+repos='bulk-memory-operations reference-types'
 
-./run.py --wasm ${V8_DIR}/test/wasm-js/data/interpreter/wasm --js ${V8_DIR}/out/x64.release/d8 --out ${SPEC_TEST_DIR}/tmp
-cp ${SPEC_TEST_DIR}/tmp/*.js ${SPEC_TEST_DIR}/tests/
-rm -rf ${SPEC_TEST_DIR}/tmp
+for repo in ${repos}; do
+  echo "Process ${repo}"
+  log_and_run cd ${TMP_DIR}
+  log_and_run git clone https://github.com/WebAssembly/${repo}
+  # Compile the spec interpreter to generate the .js test cases later.
+  log_and_run cd ${repo}/interpreter
+  log_and_run make clean opt
+  log_and_run cd ../test/core
+  log_and_run mkdir ${SPEC_TEST_DIR}/tests/proposals/${repo}
+
+  # Iterate over all proposal tests. Those which differ from the spec tests are
+  # copied to the output directory and converted to .js tests.
+  for abs_filename in ${TMP_DIR}/${repo}/test/core/*.wast; do
+    rel_filename="$(basename -- $abs_filename)"
+    test_name=${rel_filename%.wast}
+    spec_filename=${V8_DIR}/test/wasm-js/data/test/core/${rel_filename}
+    if [ ! -f "$spec_filename" ] || ! cmp -s $abs_filename $spec_filename ; then
+      log_and_run cp ${rel_filename} ${SPEC_TEST_DIR}/tests/proposals/${repo}/
+      log_and_run ./run.py --wasm ../../interpreter/wasm ${rel_filename} --out _build 2> /dev/null
+    fi
+  done
+  log_and_run cp _build/*.js ${SPEC_TEST_DIR}/tests/proposals/${repo}/
+done
+
+###############################################################################
+# Report and cleanup.
+###############################################################################
 
 cd ${SPEC_TEST_DIR}
 echo
 echo "The following files will get uploaded:"
-ls tests
+ls -R tests
 echo
 
-# For the following command you first have to authenticate with google cloud
-# storage. For that you have to execute
-#
-# > gsutil.py config
-#
-# When the script asks you for your project-id, use 0.
-upload_to_google_storage.py -a -b v8-wasm-spec-tests tests
+log_and_run rm -rf ${TMP_DIR}
+
+###############################################################################
+# Upload all spec tests.
+###############################################################################
+
+echo "****************************************************************************"
+echo "* For the following command you first have to authenticate with google cloud"
+echo "* storage. For that you have to execute"
+echo "*"
+echo "* > gsutil.py config"
+echo "*"
+echo "* When the script asks you for your project-id, use 0."
+echo "****************************************************************************"
+log_and_run cd ${SPEC_TEST_DIR}
+log_and_run upload_to_google_storage.py -a -b v8-wasm-spec-tests tests