Reviewed by Darin Adler.

        - add an option to run-webkit-tests to ignore pixel test failures where
          all pixels differ by no more than a specified threshold

        * DumpRenderTree/mac/ImageDiff.m:
        (main):
        (compareImages):
        (computePercentageDifferent):
        * Scripts/run-webkit-tests:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@27584 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/WebKitTools/ChangeLog b/WebKitTools/ChangeLog
index dd2ba1dfeea70ce6e09e07b2bea5cf227d1d5bc1..2eb016b4acfb4b9b7a8a0003257ea1cef1e1779b 100644 (file)
--- a/WebKitTools/ChangeLog
+++ b/WebKitTools/ChangeLog
@@ -1,3 +1,16 @@
+2007-11-07  Dan Bernstein  <mitz@apple.com>
+
+        Reviewed by Darin Adler.
+
+        - add an option to run-webkit-tests to ignore pixel test failures where
+          all pixels differ by no more than a specified threshold
+
+        * DumpRenderTree/mac/ImageDiff.m:
+        (main):
+        (compareImages):
+        (computePercentageDifferent):
+        * Scripts/run-webkit-tests:
+
 2007-11-07  Simon Hausmann  <hausmann@kde.org>
 
         Reviewed by Lars.
@@ -1227,7 +1240,7 @@
 
         * DumpRenderTree/win/DumpRenderTree.vcproj:
 
-2007-10-18  Dan Bernstein  <dan.bernstein@apple.com>
+2007-10-18  Dan Bernstein  <mitz@apple.com>
 
         Reviewed by Adam Roben.
 

diff --git a/WebKitTools/DumpRenderTree/mac/ImageDiff.m b/WebKitTools/DumpRenderTree/mac/ImageDiff.m
index 8ceb0bf700746c623b1a27205dc7f1c1f60b5041..e485b7d7bbec926293ac0217d19fefc16e98d90f 100644 (file)
--- a/WebKitTools/DumpRenderTree/mac/ImageDiff.m
+++ b/WebKitTools/DumpRenderTree/mac/ImageDiff.m
@@ -30,18 +30,40 @@
 #import <AppKit/NSGraphicsContext.h>
 #import <AppKit/NSCIImageRep.h>
 
+#import <getopt.h>
+
 /* prototypes */
 int main(int argc, const char *argv[]);
 CGImageRef createImageFromStdin(int imageSize);
-void compareImages(CGImageRef actualBitmap, CGImageRef baselineImage);
+void compareImages(CGImageRef actualBitmap, CGImageRef baselineImage, unsigned threshold);
 NSBitmapImageRep *getDifferenceBitmap(CGImageRef actualBitmap, CGImageRef baselineImage);
-float computePercentageDifferent(NSBitmapImageRep *diffBitmap);
+float computePercentageDifferent(NSBitmapImageRep *diffBitmap, unsigned threshold);
 
 
 int main(int argc, const char *argv[])
 {
     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
 
+    unsigned threshold = 0;
+
+    struct option options[] = {
+        {"threshold", required_argument, NULL, 't'},
+        {NULL, 0, NULL, 0}
+    };
+
+    int option;
+    while ((option = getopt_long(argc, (char * const *)argv, "", options, NULL)) != -1) {
+        switch (option) {
+        case 't':
+            threshold = strtol(optarg, NULL, 0);
+            break;
+        case '?':   // unknown or ambiguous option
+        case ':':   // missing argument
+            exit(1);
+            break;
+        }
+    }
+    
     char buffer[2048];
     CGImageRef actualImage = nil;
     CGImageRef baselineImage = nil;
@@ -67,7 +89,7 @@ int main(int argc, const char *argv[])
         }
 
         if (actualImage != nil && baselineImage != nil) {
-            compareImages(actualImage, baselineImage);
+            compareImages(actualImage, baselineImage, threshold);
             CGImageRelease(actualImage);
             CGImageRelease(baselineImage);
             actualImage = nil;
@@ -104,12 +126,12 @@ CGImageRef createImageFromStdin(int bytesRemaining)
     return image; 
 }
 
-void compareImages(CGImageRef actualBitmap, CGImageRef baselineBitmap)
+void compareImages(CGImageRef actualBitmap, CGImageRef baselineBitmap, unsigned threshold)
 {
     // prepare the difference blend to check for pixel variations
     NSBitmapImageRep *diffBitmap = getDifferenceBitmap(actualBitmap, baselineBitmap);
             
-    float percentage = computePercentageDifferent(diffBitmap);
+    float percentage = computePercentageDifferent(diffBitmap, threshold);
     
     percentage = (float)((int)(percentage * 100.0f)) / 100.0f; // round to 2 decimal places
     
@@ -158,7 +180,7 @@ NSBitmapImageRep *getDifferenceBitmap(CGImageRef testBitmap, CGImageRef referenc
  * Counts the number of non-black pixels, and returns the percentage
  * of non-black pixels to total pixels in the image.
  */
-float computePercentageDifferent(NSBitmapImageRep *diffBitmap)
+float computePercentageDifferent(NSBitmapImageRep *diffBitmap, unsigned threshold)
 {
     // if diffBiatmap is nil, then there was an error, and it didn't match.
     if (diffBitmap == nil)
@@ -180,7 +202,8 @@ float computePercentageDifferent(NSBitmapImageRep *diffBitmap)
             unsigned char* red = pixelRowData + col;
             unsigned char* green = red + 1;
             unsigned char* blue = red + 2;
-            if (*red != 0 || *green != 0 || *blue != 0) {
+            unsigned distance = *red + *green + *blue;
+            if (distance > threshold) {
                 differences++;
                 // shift the pixels towards white to make them more visible
                 *red = MIN(UCHAR_MAX, *red + 100);

diff --git a/WebKitTools/Scripts/run-webkit-tests b/WebKitTools/Scripts/run-webkit-tests
index f8eddcf2b685ed46c8def2c806379c79004954fa..76ebbe9350d2a0db5cac78fb2be856bf56e54f69 100755 (executable)
--- a/WebKitTools/Scripts/run-webkit-tests
+++ b/WebKitTools/Scripts/run-webkit-tests
@@ -111,6 +111,7 @@ my $testHTTP = 1;
 my $testMedia = 1;
 my $testResultsDirectory = "/tmp/layout-test-results";
 my $threaded = 0;
+my $threshold = 0;
 my $treatSkipped = "default";
 my $verbose = 0;
 my $useValgrind = 0;
@@ -177,6 +178,7 @@ Usage: $programName [options] [testdir|testpath ...]
   --strict                        Do a comparison with the output on Mac (Qt only)
   --[no-]strip-editing-callbacks  Remove editing callbacks from expected results
   -t|--threaded                   Run a concurrent JavaScript thead with each test
+  --threshold t                   Ignore pixel value deviations less than or equal to t
   --valgrind                      Run DumpRenderTree inside valgrind (Qt/Linux only)
   -v|--verbose                    More verbose output (overrides --quiet)
 EOF
@@ -205,6 +207,7 @@ my $getOptionsResult = GetOptions(
     'skipped=s' => \&validateSkippedArg,
     'slowest' => \$report10Slowest,
     'threaded|t' => \$threaded,
+    'threshold=i' => \$threshold,
     'verbose|v' => \$verbose,
     'valgrind' => \$useValgrind,
     'strict' => \$strictTesting,
@@ -230,6 +233,7 @@ my $configurationOption = "--" . lc($configuration);
 $repaintTests = 1 if $repaintSweepHorizontally;
 
 $pixelTests = 1 if $repaintTests;
+$pixelTests = 1 if $threshold > 0;
 
 $verbose = 1 if $testsPerDumpTool == 1;
 
@@ -415,13 +419,16 @@ push @toolArgs, "--threaded" if $threaded;
 push @toolArgs, "--paint" if $shouldCheckLeaks; # Otherwise, DRT won't exercise painting leaks.
 push @toolArgs, "-";
 
+my @diffToolArgs = ();
+push @diffToolArgs, "--threshold", $threshold;
+
 $| = 1;
 
 my $imageDiffToolPID;
 if ($pixelTests) {
     local %ENV;
     $ENV{MallocStackLogging} = 1 if $shouldCheckLeaks;
-    $imageDiffToolPID = open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, "") or die "unable to open $imageDiffTool\n";
+    $imageDiffToolPID = open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, @diffToolArgs) or die "unable to open $imageDiffTool\n";
 }
 
 my $dumpToolPID;