[test262-runner] Print results report to HTML

https://bugs.webkit.org/show_bug.cgi?id=186790

Patch by Leo Balter <leonardo.balter@gmail.com> on 2018-06-19
Reviewed by Michael Saboff.

After runing the tests, 2 html files are created with a list to failing tests and summaries.
* Scripts/test262/Runner.pm:
(main):
(summarizeResults):
(printHTML):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@232994 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index a313fe7..16cea27 100644 (file)
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,16 @@
+2018-06-19  Leo Balter  <leonardo.balter@gmail.com>
+
+        [test262-runner] Print results report to HTML
+        https://bugs.webkit.org/show_bug.cgi?id=186790
+
+        Reviewed by Michael Saboff.
+
+        After runing the tests, 2 html files are created with a list to failing tests and summaries.
+        * Scripts/test262/Runner.pm:
+        (main):
+        (summarizeResults):
+        (printHTML):
+
 2018-06-19  Keith Rollin  <krollin@apple.com>
 
         Crash running check-webkit-style on webrtc/.../exceptions.py

diff --git a/Tools/Scripts/test262/Runner.pm b/Tools/Scripts/test262/Runner.pm
index 4d9e822..ae521ba 100755 (executable)
--- a/Tools/Scripts/test262/Runner.pm
+++ b/Tools/Scripts/test262/Runner.pm
@@ -186,7 +186,6 @@ sub processCLI {
         }
     }
 
-
     if ($JSC) {
         $JSC = abs_path($JSC);
         # Make sure the path and file jsc exist
@@ -292,8 +291,11 @@ sub main {
 
     print $deffh getHarness(\@defaultHarnessFiles);
 
-    # If not commandline test path supplied, use the root directory of all tests.
-    push(@cliTestDirs, 'test') if not @cliTestDirs;
+    if (!@cliTestDirs) {
+        # If not commandline test path supplied, use the root directory of all tests.
+        push(@cliTestDirs, 'test') if not @cliTestDirs;
+        $runningAllTests = 1;
+    }
 
     if ($latestImport) {
         @files = loadImportFile();
@@ -301,7 +303,6 @@ sub main {
         # If we only want to re-run failure, only run tests in results file
         findAllFailing();
     } else {
-        $runningAllTests = 1;
         # Otherwise, get all files from directory
         foreach my $testsDir (@cliTestDirs) {
             find(
@@ -501,36 +502,38 @@ sub main {
         print "---------------------------------------------------------\n";
     }
 
-    print("\n");
+    my $totalRun = scalar @results - $skipfilecount;
+    print "\n$totalRun tests run\n";
+    print "$skipfilecount test files skipped\n";
+
+    if (!$expect) {
+        print "$failcount tests failed\n";
+        print "$newpasscount tests newly pass\n" if $skippedOnly;
+    } else {
+        print "$failcount tests failed in total\n";
+        print "$newfailcount tests newly fail\n";
+        print "$newpasscount tests newly pass\n";
+    }
 
     if ($saveExpectations) {
         DumpFile($expectationsFile, \%failed);
         print "Saved expectation file in: $expectationsFile\n";
     }
-    if ($runningAllTests) {
-        if (! -e $resultsDir) {
-            mkpath($resultsDir);
-        }
-        $resultsFile = abs_path("$resultsDir/results.yaml");
-
-        DumpFile($resultsFile, \@results);
-        print "Saved all the results in $resultsFile\n";
-        summarizeResults();
+    if (! -e $resultsDir) {
+        mkpath($resultsDir);
     }
 
-    my $total = scalar @results - $skipfilecount;
-    print "\n" . $total . " tests ran\n";
+    $resultsFile = abs_path("$resultsDir/results.yaml");
 
-    if ( !$expect ) {
-        print $failcount . " tests failed\n";
-        print $newpasscount . " tests newly pass\n" if $skippedOnly;
-    } else {
-        print $failcount . " tests failed in total\n";
-        print $newfailcount . " tests newly fail\n";
-        print $newpasscount . " tests newly pass\n";
-    }
+    DumpFile($resultsFile, \@results);
+    print "Saved all the results in $resultsFile\n";
+
+    my $styleCss = abs_path("$Bin/report.css");
+    qx/cp $styleCss $resultsDir/;
+    summarizeResults();
+    printHTMLResults(\%failed, $totalRun, $failcount, $newfailcount, $skipfilecount);
 
-    print $skipfilecount . " test files skipped\n";
+    print "See the summaries and results in the $resultsDir.\n\n";
 
     printf("Done in %.2f seconds!\n", time() - $startTime);
 
@@ -916,6 +919,7 @@ sub summarizeResults {
     }
     $summaryTxtFile = abs_path("$resultsDir/summary.txt");
     $summaryFile = abs_path("$resultsDir/summary.yaml");
+    my $summaryHTMLFile = abs_path("$resultsDir/summary.html");
 
     my %byfeature;
     my %bypath;
@@ -972,66 +976,137 @@ sub summarizeResults {
     }
 
     open(my $sfh, '>', $summaryTxtFile) or die $!;
+    open(my $htmlfh, '>', $summaryHTMLFile) or die $!;
+
+    print $htmlfh qq{<html><head>
+        <title>Test262 Summaries</title>
+        <link rel="stylesheet" href="report.css">
+        </head>
+        <body>
+        <h1>Test262 Summaries</h1>
+        <div class="visit">Visit <a href="index.html">the index</a> for a report of failures.</div>
+        <h2>By Features</h2>
+        <table class="summary-table">
+            <thead>
+                <th>Feature</th>
+                <th>%</th>
+                <th>Total</th>
+                <th>Run</th>
+                <th>Passed</th>
+                <th>Failed</th>
+                <th>Skipped</th>
+                <th>Exec. time</th>
+                <th>Avg. time</th>
+            </thead>
+            <tbody>};
+
+    print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RUN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FEATURE');
 
-    print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RAN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FOLDER');
-    foreach my $key (sort keys %bypath) {
-        my $totalFilesRan = $bypath{$key}->[0] + $bypath{$key}->[1];
-        my $totalFiles = $totalFilesRan + $bypath{$key}->[2];
+    foreach my $key (sort keys %byfeature) {
+        my $totalFilesRun = $byfeature{$key}->[0] + $byfeature{$key}->[1];
+        my $totalFiles = $totalFilesRun + $byfeature{$key}->[2];
 
-        my $per = sprintf("%.0f", ($bypath{$key}->[0] / $totalFiles) * 100) . "%";
+        my $iper = ($byfeature{$key}->[0] / $totalFiles) * 100;
+        my $per = sprintf("%.0f", $iper) . "%";
 
-        my $time = sprintf("%.1f", $bypath{$key}->[3]) . "s";
+        my $time = sprintf("%.1f", $byfeature{$key}->[3]) . "s";
         my $avgTime;
 
-        if ($totalFilesRan) {
-            $avgTime = sprintf("%.2f", $bypath{$key}->[3] / $totalFilesRan) . "s";
+        if ($totalFilesRun) {
+            $avgTime = sprintf("%.2f", $byfeature{$key}->[3] / $totalFilesRun) . "s";
         } else {
             $avgTime = "0s";
         }
 
         print $sfh sprintf("%-6s %-6s %-6s %-6d %-6d %-6d %-7s %-6s %s\n",
                            $totalFiles,
-                           $totalFilesRan,
+                           $totalFilesRun,
                            $per,
-                           $bypath{$key}->[0],
-                           $bypath{$key}->[1],
-                           $bypath{$key}->[2],
+                           $byfeature{$key}->[0],
+                           $byfeature{$key}->[1],
+                           $byfeature{$key}->[2],
                            $time,
                            $avgTime,
                            $key);
-    }
 
-    print $sfh "\n\n";
-    print $sfh sprintf("%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RAN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FEATURE');
+        print $htmlfh qq{
+            <tr class="per-$iper">
+                <td>$key</td>
+                <td>$per</td>
+                <td>$totalFiles</td>
+                <td>$totalFilesRun</td>
+                <td>$byfeature{$key}->[0]</td>
+                <td>$byfeature{$key}->[1]</td>
+                <td>$byfeature{$key}->[2]</td>
+                <td>$time</td>
+                <td>$avgTime</td>
+            </tr>};
+    }
 
-    foreach my $key (sort keys %byfeature) {
-        my $totalFilesRan = $byfeature{$key}->[0] + $byfeature{$key}->[1];
-        my $totalFiles = $totalFilesRan + $byfeature{$key}->[2];
+    print $htmlfh qq{</tbody></table>
+        <h2>By Path</h2>
+        <table class="summary-table">
+            <thead>
+                <th>Folder</th>
+                <th>%</th>
+                <th>Total</th>
+                <th>Run</th>
+                <th>Passed</th>
+                <th>Failed</th>
+                <th>Skipped</th>
+                <th>Exec. time</th>
+                <th>Avg. time</th>
+            </thead>
+            <tbody>};
+
+    print $sfh sprintf("\n\n%-6s %-6s %-6s %-6s %-6s %-6s %-7s %-6s %s\n", 'TOTAL', 'RUN', 'PASS-%', 'PASS', 'FAIL', 'SKIP', 'TIME', 'AVG', 'FOLDER'); 
+    foreach my $key (sort keys %bypath) {
+        my $totalFilesRun = $bypath{$key}->[0] + $bypath{$key}->[1];
+        my $totalFiles = $totalFilesRun + $bypath{$key}->[2];
 
-        my $per = sprintf("%.0f", ($byfeature{$key}->[0] / $totalFiles) * 100) . "%";
+        my $iper = ($bypath{$key}->[0] / $totalFiles) * 100;
+        my $per = sprintf("%.0f", $iper) . "%";
 
-        my $time = sprintf("%.1f", $byfeature{$key}->[3]) . "s";
+        my $time = sprintf("%.1f", $bypath{$key}->[3]) . "s";
         my $avgTime;
 
-        if ($totalFilesRan) {
-            $avgTime = sprintf("%.2f", $byfeature{$key}->[3] / $totalFilesRan) . "s";
+        if ($totalFilesRun) {
+            $avgTime = sprintf("%.2f", $bypath{$key}->[3] / $totalFilesRun) . "s";
         } else {
             $avgTime = "0s";
         }
 
         print $sfh sprintf("%-6s %-6s %-6s %-6d %-6d %-6d %-7s %-6s %s\n",
                            $totalFiles,
-                           $totalFilesRan,
+                           $totalFilesRun,
                            $per,
-                           $byfeature{$key}->[0],
-                           $byfeature{$key}->[1],
-                           $byfeature{$key}->[2],
+                           $bypath{$key}->[0],
+                           $bypath{$key}->[1],
+                           $bypath{$key}->[2],
                            $time,
                            $avgTime,
                            $key);
+
+        print $htmlfh qq{
+            <tr class="per-$iper">
+                <td>$key</td>
+                <td>$per</td>
+                <td>$totalFiles</td>
+                <td>$totalFilesRun</td>
+                <td>$bypath{$key}->[0]</td>
+                <td>$bypath{$key}->[1]</td>
+                <td>$bypath{$key}->[2]</td>
+                <td>$time</td>
+                <td>$avgTime</td>
+            </tr>};
     }
 
+    print $htmlfh qq{</tbody></table>
+        <div class="visit">Visit <a href="index.html">the index</a> for a report of failures.</div>
+        </body></html>};
+
     close($sfh);
+    close($htmlfh);
 
     my %resultsyaml = (
         byFolder => \%bypath,
@@ -1039,8 +1114,6 @@ sub summarizeResults {
     );
 
     DumpFile($summaryFile, \%resultsyaml);
-
-    print "See summarized results in $summaryTxtFile\n";
 }
 
 sub findAllFailing {
@@ -1057,6 +1130,57 @@ sub findAllFailing {
     @files = map { qq($test262Dir/$_) } keys %filedictionary;
 }
 
+sub printHTMLResults {
+    my %failed = %{shift()};
+    my ($total, $failcount, $newfailcount, $skipcount) = @_;
+
+    # Create test262-results folder if it does not exits
+    if (! -e $resultsDir) {
+        mkpath($resultsDir);
+    }
+
+    my $indexHTML = abs_path("$resultsDir/index.html");
+    open(my $htmlfh, '>', $indexHTML) or die $!;
+
+    print $htmlfh qq{<html><head>
+        <title>Test262 Results</title>
+        <link rel="stylesheet" href="report.css">
+        </head>
+        <body>
+        <h1>Test262 Results</h1>
+        <div class="visit">Visit <a href="summary.html">the summary</a> for statistics.</div>};
+
+    print $htmlfh qq{<h2>Stats</h2><ul>};
+
+    {
+        my $failedFiles = scalar (keys %failed);
+        my $totalPlus = $total + $skipcount;
+        print $htmlfh qq{
+            <li>$total test files run from $totalPlus files, $skipcount skipped test files</li>
+            <li>$failcount failures from $failedFiles distinct files, $newfailcount new failures</li>
+        };
+    }
+
+    print $htmlfh qq{</ul><h2>Failures</h2><ul>};
+
+    while (my ($path, $scenarios) = each %failed) {
+        print $htmlfh qq{<li class="list-item">
+            <label for="$path" class="expander-control">$path</label>
+            <input type="checkbox" id="$path" class="expander">
+            <ul class="expand">};
+        while (my ($scenario, $value) = each %{$scenarios}) {
+            print $htmlfh qq{<li>$scenario: $value</li>};
+        }
+        print $htmlfh qq{</ul></li>}
+    }
+
+    print $htmlfh qq{</ul>
+    <div class="visit">Visit <a href="summary.html">the summary</a> for statistics.</div>
+    </body></html>};
+
+    close $htmlfh;
+}
+
 __END__
 
 =head1 DESCRIPTION

diff --git a/Tools/Scripts/test262/report.css b/Tools/Scripts/test262/report.css
new file mode 100644 (file)
index 0000000..30edebe
--- /dev/null
+++ b/Tools/Scripts/test262/report.css
@@ -0,0 +1,47 @@
+html {
+    line-height: 1.3em;
+}
+
+.visit {
+    margin: 1em 0;
+}
+
+.expander, .expand {
+    display: none;
+}
+
+.expander:checked + .expand {
+    display: block;
+}
+
+.expander-control {
+    text-decoration: underline;
+    cursor: pointer;
+}
+
+.list-item {
+    line-height: 1.5em;
+}
+
+.summary-table {
+    border-collapse: collapse;
+}
+
+.summary-table th, .summary-table td{
+    text-align: left;
+    padding: 0 .4em;
+    line-height: 2em;
+    border: 1px solid #333;
+}
+
+.summary-table tr:nth-child(even) td {
+    box-shadow: inset 100em 1em rgba(0,0,0,.04);
+}
+
+.per-100 {
+    background-color: #e8ffde;
+}
+
+.per-0 {
+    background-color: #fac8c8;
+}
\ No newline at end of file