WebAssembly: add script which can import GCC torture tests
authorjfbastien@apple.com <jfbastien@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 19 Apr 2017 18:02:04 +0000 (18:02 +0000)
committerjfbastien@apple.com <jfbastien@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 19 Apr 2017 18:02:04 +0000 (18:02 +0000)
https://bugs.webkit.org/show_bug.cgi?id=170740

Reviewed by Saam Barati.

Add a script which can import the GCC torture tests and create a
yaml file to run them as part of jsc's WebAssembly regression
tests.

This patch doesn't commit the tests themselves because they're
licensed differently.

* Scripts/run-jsc-stress-tests: learn how to run
Emscripten-generated .js+.wasm files, and do a bit of cleanup on
the options because WebAssembly is enabled by default.
* Scripts/update-wasm-gcc-torture.py: Added.
(parse_args):
(update_lkgr):
(untar_torture):
(list_js_files):
(waterfall_known_failures):
(create_yaml):
(main):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215519 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/run-jsc-stress-tests
Tools/Scripts/update-wasm-gcc-torture.py [new file with mode: 0755]

index 2ae9830..a1d5813 100644 (file)
@@ -1,3 +1,29 @@
+2017-04-19  JF Bastien  <jfbastien@apple.com>
+
+        WebAssembly: add script which can import GCC torture tests
+        https://bugs.webkit.org/show_bug.cgi?id=170740
+
+        Reviewed by Saam Barati.
+
+        Add a script which can import the GCC torture tests and create a
+        yaml file to run them as part of jsc's WebAssembly regression
+        tests.
+
+        This patch doesn't commit the tests themselves because they're
+        licensed differently.
+
+        * Scripts/run-jsc-stress-tests: learn how to run
+        Emscripten-generated .js+.wasm files, and do a bit of cleanup on
+        the options because WebAssembly is enabled by default.
+        * Scripts/update-wasm-gcc-torture.py: Added.
+        (parse_args):
+        (update_lkgr):
+        (untar_torture):
+        (list_js_files):
+        (waterfall_known_failures):
+        (create_yaml):
+        (main):
+
 2017-04-19  Youenn Fablet  <youenn@apple.com>
 
         Import web-platform-tests/tools
index 1d49f4a..04e9880 100755 (executable)
@@ -1198,7 +1198,19 @@ def runWebAssembly
     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
-    run("default-wasm", "-m", "--useWebAssembly=1")
+    run("default-wasm", "-m")
+end
+
+def runWebAssemblyEmscripten(mode)
+    case mode
+    when :skip
+        return
+    end
+    return if !$jitTests
+    return if !$isFTLPlatform
+    wasm = $benchmark.to_s.sub! '.js', '.wasm'
+    prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
+    run("default-wasm")
 end
 
 def runWebAssemblySpecTest(mode)
@@ -1217,7 +1229,7 @@ def runWebAssemblySpecTest(mode)
     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
 
-    runWithOutputHandler("default-wasm", noisyOutputHandler, "--useWebAssembly=1", "../spec-harness.js")
+    runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js")
 end
 
 def runChakra(mode, exception, baselineFile, extraFiles)
diff --git a/Tools/Scripts/update-wasm-gcc-torture.py b/Tools/Scripts/update-wasm-gcc-torture.py
new file mode 100755 (executable)
index 0000000..9a3597b
--- /dev/null
@@ -0,0 +1,148 @@
+#! /usr/bin/env python
+#
+# Copyright (C) 2017 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import argparse
+import glob
+import json
+import os.path
+import shutil
+import subprocess
+import sys
+import tarfile
+import urllib2
+
+LKGR_URL = 'https://storage.googleapis.com/wasm-llvm/builds/%s/lkgr.json'
+TORTURE_FILE = 'wasm-torture-emwasm-%s.tbz2'
+TORTURE_URL = 'https://storage.googleapis.com/wasm-llvm/builds/%s/%s/%s'
+TORTURE_DIR = 'emwasm-torture-out'
+WATERFALL_DIR = 'waterfall'
+WATERFALL_GIT = 'https://github.com/WebAssembly/waterfall.git'
+WATERFALL_KNOWN_FAILURES = 'src/test/run_known_gcc_test_failures.txt'
+
+DESCRIPTION = """GCC torture test packager for jsc testing
+
+Obtain the GCC torture tests as compiled for WebAssembly by the waterfall, using
+Emscripten with its embedded libc. Create a yaml file which can be executed with
+run-jsc-stress-tests."""
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=DESCRIPTION,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog='See http://wasm-stat.us for current waterfall status')
+    parser.add_argument('--lkgr', default=None, help='LKGR value to use')
+    parser.add_argument('--platform', default='mac', help='Waterfall platform to sync to')
+    parser.add_argument('--nountar', default=False, action='store_true', help='If set, torture tests are assumed to already be in their destination folder')
+    parser.add_argument('--waterfall_hash', default=None, help='git hash to use for the waterfall repo')
+    parser.add_argument('--yaml', default='gcc-torture.yaml', help='yaml test file to generate')
+    return parser.parse_args()
+
+
+def update_lkgr(args):
+    lkgr_url = LKGR_URL % args.platform
+    if not args.lkgr:
+        print 'Downloading: %s' % lkgr_url
+        args.lkgr = json.loads(urllib2.urlopen(lkgr_url).read())['build']
+    print 'lkgr: %s' % args.lkgr
+
+
+def untar_torture(args):
+    torture_file = TORTURE_FILE % args.lkgr
+    torture_url = TORTURE_URL % (args.platform, args.lkgr, torture_file)
+    if not os.path.exists(torture_file):
+        print 'Downloading: %s' % torture_url
+        torture_download = urllib2.urlopen(torture_url)
+        with open(torture_file, 'wb') as f:
+            f.write(torture_download.read())
+    if not args.nountar:
+        if os.path.isdir(TORTURE_DIR):
+            shutil.rmtree(TORTURE_DIR)
+        with tarfile.open(torture_file, 'r') as torture_tar:
+            print 'Extracting: %s -> %s' % (torture_file, TORTURE_DIR)
+            torture_tar.extractall()
+    assert os.path.isdir(TORTURE_DIR)
+
+
+def list_js_files(args):
+    js_files = sorted([os.path.basename(f) for f in glob.glob(os.path.join(TORTURE_DIR, '*.js'))])
+    print 'Found %s JavaScript tests' % len(js_files)
+    assert len(js_files) > 1200
+    return js_files
+
+
+def waterfall_known_failures(args):
+    if not os.path.isdir(WATERFALL_DIR):
+        subprocess.check_call(['git', 'clone', WATERFALL_GIT, WATERFALL_DIR])
+    else:
+        subprocess.check_call(['git', 'pull'], cwd=WATERFALL_DIR)
+    if args.waterfall_hash:
+        subprocess.check_call(['git', 'checkout', args.waterfall_hash], cwd=WATERFALL_DIR)
+    else:
+        args.waterfall_hash = subprocess.check_output(['git', 'log', '-n1', '--pretty=format:%H'], cwd=WATERFALL_DIR).strip()
+    print 'Waterfall at: %s' % args.waterfall_hash
+    known_failures = []
+    with open(os.path.join(WATERFALL_DIR, WATERFALL_KNOWN_FAILURES)) as failures_file:
+        for line in failures_file:
+            line = line.strip()
+            if '#' in line:
+                line = line[:line.index('#')].strip()
+            tokens = line.split()
+            if not tokens:
+                continue
+            test_file = tokens[0]
+            if len(tokens) > 1:
+                attributes = set(tokens[1].split(','))
+                if 'emwasm' not in attributes:
+                    continue
+            if os.path.splitext(test_file)[-1] != '.js':
+                continue
+            known_failures.append(test_file)
+    assert known_failures
+    return set(known_failures)
+
+
+def create_yaml(args, js_files, known_failures):
+    with open(args.yaml, 'w+') as yaml:
+        yaml.write('# GCC torture tests\n')
+        yaml.write('# Auto-generated by: %s\n' % sys.argv[0])
+        yaml.write('# lkgr: %s\n' % args.lkgr)
+        yaml.write('# Waterfall hash: %s\n' % args.waterfall_hash)
+        for js in js_files:
+            yaml.write('\n- path: %s/%s\n' % (TORTURE_DIR, js))
+            yaml.write('  cmd: runWebAssemblyEmscripten :%s\n' % ('normal' if js not in known_failures else 'skip'))
+
+
+def main():
+    args = parse_args()
+    update_lkgr(args)
+    untar_torture(args)
+    js_files = list_js_files(args)
+    known_failures = waterfall_known_failures(args)
+    create_yaml(args, js_files, known_failures)
+
+
+if __name__ == '__main__':
+    sys.exit(main())