Unreviewed, rolling out r232112.
authorryanhaddad@apple.com <ryanhaddad@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 23 May 2018 16:59:58 +0000 (16:59 +0000)
committerryanhaddad@apple.com <ryanhaddad@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 23 May 2018 16:59:58 +0000 (16:59 +0000)
The tests added with this change  are failing on the bots.

Reverted changeset:

"test262/Runner.pm: add unit tests"
https://bugs.webkit.org/show_bug.cgi?id=185783
https://trac.webkit.org/changeset/232112

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@232115 268f45cc-cd09-0410-ab3c-d52691b4dbfc

12 files changed:
Tools/ChangeLog
Tools/Scripts/test262/Runner.pm
Tools/Scripts/webkitperl/test262_unittest/README [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js [deleted file]
Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js [deleted file]
Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl [deleted file]

index 64e5a1d..dbb5e7c 100644 (file)
@@ -1,3 +1,15 @@
+2018-05-23  Ryan Haddad  <ryanhaddad@apple.com>
+
+        Unreviewed, rolling out r232112.
+
+        The tests added with this change  are failing on the bots.
+
+        Reverted changeset:
+
+        "test262/Runner.pm: add unit tests"
+        https://bugs.webkit.org/show_bug.cgi?id=185783
+        https://trac.webkit.org/changeset/232112
+
 2018-05-23  Valerie R Young  <valerie@bocoup.com>
 
         test262/Runner.pm: add unit tests
index a4174b7..ec30d01 100755 (executable)
@@ -78,6 +78,7 @@ my $max_process;
 my @cliTestDirs;
 my $verbose;
 my $JSC;
+my $test262Dir;
 my $harnessDir;
 my %filterFeatures;
 my $ignoreConfig;
@@ -90,15 +91,15 @@ my $latestImport;
 my $runningAllTests;
 my $timeout;
 
-my $test262Dir;
-my $webkitTest262Dir = abs_path("$Bin/../../../JSTests/test262");
 my $expectationsFile = abs_path("$Bin/../../../JSTests/test262/expectations.yaml");
 my $configFile = abs_path("$Bin/../../../JSTests/test262/config.yaml");
 
 my $resultsDir = $ENV{PWD} . "/test262-results";
-my $resultsFile;
-my $summaryTxtFile;
-my $summaryFile;
+mkpath($resultsDir);
+
+my $resultsFile = abs_path("$resultsDir/results.yaml");
+my $summaryTxtFile = abs_path("$resultsDir/summary.txt");
+my $summaryFile = abs_path("$resultsDir/summary.yaml");
 
 my @results;
 my @files;
@@ -106,6 +107,8 @@ my @files;
 my $tempdir = tempdir();
 my ($deffh, $deffile) = getTempFile();
 
+my @default_harnesses;
+
 my $startTime = time();
 
 main();
@@ -163,26 +166,15 @@ sub processCLI {
         }
     }
 
-    if ($stats || $failingOnly) {
-        # If not supplied, try to find the results file in expected directory
-        $resultsFile ||= abs_path("$resultsDir/results.yaml");
-
-        if ($failingOnly && ! -e $resultsFile) {
-            die "Error: cannot find results file to run failing tests," .
+    if ($stats) {
+        if (! -e $resultsFile) {
+            die "Error: cannot find results file to summarize," .
                 "please specify with --results.";
         }
-
-        if ($stats) {
-            if (! -e $resultsFile) {
-                die "Error: cannot find results file to summarize," .
-                    " please specify with --results.";
-            }
-            summarizeResults();
-            exit;
-        }
+        summarizeResults();
+        exit;
     }
 
-
     if ($JSC) {
         $JSC = abs_path($JSC);
         # Make sure the path and file jsc exist
@@ -203,18 +195,11 @@ sub processCLI {
     }
 
     if (! $test262Dir) {
-        $test262Dir = $webkitTest262Dir;
+        $test262Dir = abs_path("$Bin/../../../JSTests/test262");
     } else {
         $test262Dir = abs_path($test262Dir);
     }
-
     $harnessDir = "$test262Dir/harness";
-    if (! -e $harnessDir) {
-        # if the harness directory does not exist in the custom test262 path,
-        # then use the webkits harness directory.
-        $harnessDir = "$webkitTest262Dir/harness";
-    }
-
 
     if (! $ignoreConfig) {
         if ($configFile && ! -e $configFile) {
@@ -227,6 +212,11 @@ sub processCLI {
         }
     }
 
+    if ( $failingOnly && ! -e $resultsFile ) {
+        die "Error: cannot find results file to run failing tests," .
+            " please specify with --results.";
+    }
+
     if ($specifiedExpectationsFile) {
         $expectationsFile = abs_path($specifiedExpectationsFile);
         if (! -e $expectationsFile && ! $ignoreExpectations) {
@@ -266,18 +256,16 @@ sub processCLI {
     print "--------------------------------------------------------\n\n";
 }
 
-
 sub main {
     processCLI();
 
-    my @defaultHarnessFiles = (
+    @default_harnesses = (
         "$harnessDir/sta.js",
         "$harnessDir/assert.js",
         "$harnessDir/doneprintHandle.js",
         "$Bin/agent.js"
     );
-
-    print $deffh getHarness(\@defaultHarnessFiles);
+    print $deffh getHarness(<@default_harnesses>);
 
     # If not commandline test path supplied, use the root directory of all tests.
     push(@cliTestDirs, 'test') if not @cliTestDirs;
@@ -401,11 +389,6 @@ sub main {
     }
 
     if ($runningAllTests) {
-        if (! -e $resultsDir) {
-            mkpath($resultsDir);
-        }
-        $resultsFile = abs_path("$resultsDir/results.yaml");
-
         DumpFile($resultsFile, \@results);
         print "Saved all the results in $resultsFile\n";
         summarizeResults();
@@ -609,8 +592,7 @@ sub compileTest {
     my $includes = shift;
     my ($tfh, $tfname) = getTempFile();
 
-    my @includes = map { "$harnessDir/$_" } @{ $includes };
-    my $includesContent = getHarness(\@includes);
+    my $includesContent = getHarness(map { "$harnessDir/$_" } @{ $includes });
     print $tfh $includesContent;
 
     return ($tfh, $tfname);
@@ -682,11 +664,10 @@ sub processResult {
 
     if ($scenario ne 'skip' && $currentfailure) {
 
-        # We have a new failure if we haven't loaded an expectation file
-        # (all fails are new) OR we have loaded an expectation fail and
-        # (there is no expected failure OR the failure has changed).
-        my $isnewfailure = ! $expect
-            || !$expectedfailure || $expectedfailure ne $currentfailure;
+        # We have a new failure if we have loaded an expectation file
+        # AND (there is no expected failure OR the failure has changed).
+        my $isnewfailure = $expect
+            && (!$expectedfailure || $expectedfailure ne $currentfailure);
 
         # Print the failure if we haven't loaded an expectation file
         # or the failure is new.
@@ -762,10 +743,9 @@ sub parseData {
 }
 
 sub getHarness {
-    my ($filesref) = @_;
-
+    my @files = @_;
     my $content;
-    for (@{$filesref}) {
+    for (@files) {
         my $file = $_;
 
         open(my $harness_file, '<', $file)
@@ -776,7 +756,7 @@ sub getHarness {
         close $harness_file;
     };
 
-    return $content || '';
+    return $content;
 }
 
 sub summarizeResults {
@@ -786,14 +766,6 @@ sub summarizeResults {
         my @rawresults = LoadFile($resultsFile) or die $!;
         @results = @{$rawresults[0]};
     }
-
-    # Create test262-results folder if it does not exits
-    if (! -e $resultsDir) {
-        mkpath($resultsDir);
-    }
-    $summaryTxtFile = abs_path("$resultsDir/summary.txt");
-    $summaryFile = abs_path("$resultsDir/summary.yaml");
-
     my %byfeature;
     my %bypath;
 
diff --git a/Tools/Scripts/webkitperl/test262_unittest/README b/Tools/Scripts/webkitperl/test262_unittest/README
deleted file mode 100644 (file)
index f5c6ed4..0000000
+++ /dev/null
@@ -1,6 +0,0 @@
-These unit tests test:
-  webkit/Tools/Scripts/test262-runner
-  webkit/Tools/Scripts/test262-import
-
-Helper perl modules for these scripts are located in:
-  webkit/Tools/Scripts/test262/
\ No newline at end of file
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml b/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml
deleted file mode 100644 (file)
index e1b85d4..0000000
+++ /dev/null
@@ -1,4 +0,0 @@
----
-test/fail.js:
-  default: 'Test262: This test fails.'
-  strict mode: 'Test262: This test fails.'
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml b/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml
deleted file mode 100644 (file)
index b662d4e..0000000
+++ /dev/null
@@ -1,10 +0,0 @@
----
-test/expected-to-fail-now-passing.js:
-  default: 'Test262: This test fails.'
-  strict mode: 'Test262: This test fails.'
-test/expected-to-fail-now-failing.js:
-  default: 'Test262: This test fails.'
-  strict mode: 'Test262: This test fails.'
-test/expected-to-fail-now-failing-with-new-error.js:
-  default: 'Test262: This test fails.'
-  strict mode: 'Test262: This test fails.'
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js
deleted file mode 100644 (file)
index 9c6cbab..0000000
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (C) 2018 Valerie Young. All rights reserved.
-// This code is governed by the BSD license found in the LICENSE file.
-/*---
-esid: sec-function-definitions
-description: Minimal test
----*/
-
-throw "Test262Unexpected: This test fails WITH AN UNEXPECTED ERROR.";
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js
deleted file mode 100644 (file)
index 712d5a1..0000000
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (C) 2018 Valerie Young. All rights reserved.
-// This code is governed by the BSD license found in the LICENSE file.
-/*---
-esid: sec-function-definitions
-description: Minimal test
----*/
-
-throw "Test262: This test fails.";
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js
deleted file mode 100644 (file)
index fa637be..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright (C) 2018 Valerie Young. All rights reserved.
-// This code is governed by the BSD license found in the LICENSE file.
-/*---
-esid: sec-function-definitions
-description: Minimal test
----*/
-
-function f() {};
-
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js
deleted file mode 100644 (file)
index 712d5a1..0000000
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (C) 2018 Valerie Young. All rights reserved.
-// This code is governed by the BSD license found in the LICENSE file.
-/*---
-esid: sec-function-definitions
-description: Minimal test
----*/
-
-throw "Test262: This test fails.";
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js
deleted file mode 100644 (file)
index 712d5a1..0000000
+++ /dev/null
@@ -1,8 +0,0 @@
-// Copyright (C) 2018 Valerie Young. All rights reserved.
-// This code is governed by the BSD license found in the LICENSE file.
-/*---
-esid: sec-function-definitions
-description: Minimal test
----*/
-
-throw "Test262: This test fails.";
diff --git a/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js b/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js
deleted file mode 100644 (file)
index fa637be..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright (C) 2018 Valerie Young. All rights reserved.
-// This code is governed by the BSD license found in the LICENSE file.
-/*---
-esid: sec-function-definitions
-description: Minimal test
----*/
-
-function f() {};
-
diff --git a/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl b/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl
deleted file mode 100755 (executable)
index 975d05b..0000000
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/usr/bin/env perl
-
-# Copyright (C) 2018 Bocoup LLC. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-#
-# 1. Redistributions of source code must retain the above
-#    copyright notice, this list of conditions and the following
-#    disclaimer.
-# 2. Redistributions in binary form must reproduce the above
-#    copyright notice, this list of conditions and the following
-#    disclaimer in the documentation and/or other materials
-#    provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
-# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
-# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-
-use strict;
-use warnings;
-
-use Test::More;
-use File::Spec;
-use FindBin;
-use Cwd qw(abs_path);
-use File::Path qw(rmtree);
-use File::Temp qw(tempfile);
-use File::Compare qw(compare);
-
-# This test should not be run on Windows
-if ($^O eq 'MSWin32' || $^O eq 'cygwin' || $^O eq 'WinCairo') {
-    plan(tests => 1);
-    is(1, 1, 'do nothing for Windows builds.');
-    exit 0;
-}
-
-my $Bin = $FindBin::Bin;
-my $runner = abs_path("$Bin/../../test262-runner");
-my $mockTest262 = abs_path("$Bin/fixtures");
-
-my @testCases = (
-    {
-        "TESTNAME" => "test262 test failed, ignore expectations",
-        "EXITSTATUS" => 1,
-        "TEST262TESTS_ARG" => "--test-only test/fail.js",
-        "EXPECTATION_ARG" => "--ignore-expectations",
-        "EXPECTED_NEW_FAIL_COUNT" => 2,
-    },
-    {
-        "TESTNAME" => "test262 test passed, ignore expectations",
-        "EXITSTATUS" => 0,
-        "TEST262TESTS_ARG" => "--test-only test/pass.js",
-        "EXPECTATION_ARG" => "--ignore-expectations",
-        "EXPECTED_NEW_FAIL_COUNT" => 0,
-    },
-    {
-        "TESTNAME" => "test262 tests newly failed",
-        "EXITSTATUS" => 1,
-        "TEST262TESTS_ARG" => "--test-only test/expected-to-pass-now-failing.js",
-        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
-        "EXPECTED_NEW_FAIL_COUNT" => 2,
-    },
-    {
-        "TESTNAME" => "test262 tests newly passed",
-        "EXITSTATUS" => 0,
-        "TEST262TESTS_ARG" => "--test-only test/expected-to-fail-now-passing.js",
-        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
-        "EXPECTED_NEW_FAIL_COUNT" => 0,
-    },
-    {
-        "TESTNAME" => "test262 tests fails, expected failure",
-        "EXITSTATUS" => 0,
-        "TEST262TESTS_ARG" => "--test-only  test/expected-to-fail-now-failing.js",
-        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
-        "EXPECTED_NEW_FAIL_COUNT" => 0,
-    },
-    {
-        "TESTNAME" => "test262 tests fails, with unexpected error string",
-        "EXITSTATUS" => 1,
-        "TEST262TESTS_ARG" => "--test-only test/expected-to-fail-now-failing-with-new-error.js",
-        "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
-        "EXPECTED_NEW_FAIL_COUNT" => 2,
-    },
-);
-
-my $testCasesCount = (scalar(@testCases) * 2) + 1;
-plan(tests => $testCasesCount);
-
-## Test error codes and expected output messages of tests ##
-
-foreach my $testcase (@testCases) {
-
-    my $test = $testcase->{TEST262TESTS_ARG};
-    my $expectation = $testcase->{EXPECTATION_ARG};
-    my $test262loc = "--t262 $mockTest262";
-
-    my $cmd = qq($runner $test262loc $test $expectation);
-    my $output = qx($cmd);
-
-    # Test the resulting exit code
-    my $exitcode = $? >> 8;
-    my $expectedexitcode = $testcase->{EXITSTATUS};
-    my $testname =  $testcase->{TESTNAME} . " (exit code: $expectedexitcode)";
-    is($exitcode, $testcase->{EXITSTATUS}, $testname);
-
-    # Test the number of occurences of string "! NEW FAIL"
-    my @newfailcount = $output =~ /! NEW FAIL/g;
-    my $expectednewfailures = $testcase->{EXPECTED_NEW_FAIL_COUNT};
-    $testname =  $testcase->{TESTNAME} . " (new failures: $expectednewfailures)";
-    is(scalar(@newfailcount), $expectednewfailures, $testname);
-}
-
-## Test format of saved expectations file ##
-
-my $test262loc = "--t262 $mockTest262";
-my ($expectationsfh, $expectationsfile) = tempfile();
-my $expect = "--expectations $expectationsfile";
-my $test = "--test-only test/fail.js";
-my $cmd = qq($runner --save --ignore-expectations $test262loc $test $expect);
-qx($cmd);
-
-my $expectedexpectationsfile = "$Bin/fixtures/expectations-compare.yaml";
-my $filesmatch = compare($expectationsfile, $expectedexpectationsfile);
-ok($filesmatch == 0, "expectations yaml file format");
-
-close $expectationsfh;
-
-END {
-    # Clean up test262 results directory after running tests
-    my $resultsDir = $ENV{PWD} . "/test262-results";
-    if (-e $resultsDir) {
-        rmtree($resultsDir);
-    }
-}