[WinCairo] httpd service install needs to precede server start
[WebKit-https.git] / Tools / Scripts / webkitpy / port / test.py
index 362a0e877900b2f9dcc78111d4e77227bcb8acf9..df5e67d068884b8aa84d35305509c06b373f2fb6 100644 (file)
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import base64
-import sys
 import time
 
 from webkitpy.port import Port, Driver, DriverOutput
-from webkitpy.port.base import VirtualTestSuite
 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
-from webkitpy.common.system.filesystem_mock import MockFileSystem
 from webkitpy.common.system.crashlogs import CrashLogs
+from webkitpy.common.version_name_map import PUBLIC_TABLE, VersionNameMap
 
 
 # This sets basic expectations for a test. Each individual expectation
@@ -100,14 +97,18 @@ class TestList(object):
 #
 # These numbers may need to be updated whenever we add or delete tests.
 #
-TOTAL_TESTS = 106
-TOTAL_SKIPS = 28
+TOTAL_TESTS = 72
+TOTAL_SKIPS = 9
 TOTAL_RETRIES = 14
 
-UNEXPECTED_PASSES = 6
+UNEXPECTED_PASSES = 7
 UNEXPECTED_FAILURES = 17
 
+
 def unit_test_list():
+    silent_audio = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+    silent_audio_with_single_bit_difference = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+    audio2 = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
     tests = TestList()
     tests.add('failures/expected/crash.html', crash=True)
     tests.add('failures/expected/exception.html', exception=True)
@@ -121,7 +122,7 @@ def unit_test_list():
               actual_checksum='image_checksum_fail-checksum',
               actual_image='image_checksum_fail-png')
     tests.add('failures/expected/audio.html',
-              actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
+              actual_audio=silent_audio, expected_audio=audio2,
               actual_text=None, expected_text=None,
               actual_image=None, expected_image=None,
               actual_checksum=None)
@@ -167,6 +168,9 @@ layer at (0,0) size 800x34
               actual_text='text-image-checksum_fail-txt',
               actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
               actual_checksum='text-image-checksum_fail-checksum')
+    tests.add('failures/unexpected/text-image-missing.html',
+              actual_text='text-image-checksum_fail-txt',
+              expected_image=None)
     tests.add('failures/unexpected/checksum-with-matching-image.html',
               actual_checksum='text-image-checksum_fail-checksum')
     tests.add('failures/unexpected/skip_pass.html')
@@ -179,7 +183,12 @@ layer at (0,0) size 800x34
     tests.add('passes/error.html', error='stuff going to stderr')
     tests.add('passes/image.html')
     tests.add('passes/audio.html',
-              actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
+              actual_audio=silent_audio, expected_audio=silent_audio,
+              actual_text=None, expected_text=None,
+              actual_image=None, expected_image=None,
+              actual_checksum=None)
+    tests.add('passes/audio-tolerance.html',
+              actual_audio=silent_audio_with_single_bit_difference, expected_audio=silent_audio,
               actual_text=None, expected_text=None,
               actual_image=None, expected_image=None,
               actual_checksum=None)
@@ -233,7 +242,7 @@ layer at (0,0) size 800x34
 
     # For testing test are properly included from platform directories.
     tests.add('platform/test-mac-leopard/http/test.html')
-    tests.add('platform/test-win-win7/http/test.html')
+    tests.add('platform/test-win-7sp0/http/test.html')
 
     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
     tests.add('platform/test-snow-leopard/http/test.html')
@@ -251,11 +260,6 @@ layer at (0,0) size 800x34
         actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
         expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png')
 
-    # For testing that virtual test suites don't expand names containing themselves
-    # See webkit.org/b/97925 and base_unittest.PortTest.test_tests().
-    tests.add('passes/test-virtual-passes.html')
-    tests.add('passes/passes/test-virtual-passes.html')
-
     return tests
 
 
@@ -340,7 +344,6 @@ Bug(test) passes/skipped/skip.html [ Skip ]
         add_file(test, '-expected.txt', test.expected_text)
         add_file(test, '-expected.png', test.expected_image)
 
-    filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg')
     # Clear the list of written files so that we can watch what happens during testing.
     filesystem.clear_written_files()
 
@@ -353,7 +356,7 @@ class TestPort(Port):
     ALL_BASELINE_VARIANTS = (
         'test-linux-x86_64',
         'test-mac-snowleopard', 'test-mac-leopard',
-        'test-win-vista', 'test-win-win7', 'test-win-xp',
+        'test-win-vista', 'test-win-7sp0', 'test-win-xp',
     )
 
     @classmethod
@@ -369,21 +372,16 @@ class TestPort(Port):
         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/TestExpectations'
         self._results_directory = None
 
-        self._operating_system = 'mac'
-        if self._name.startswith('test-win'):
-            self._operating_system = 'win'
-        elif self._name.startswith('test-linux'):
-            self._operating_system = 'linux'
-
-        version_map = {
-            'test-win-xp': 'xp',
-            'test-win-win7': 'win7',
-            'test-win-vista': 'vista',
-            'test-mac-leopard': 'leopard',
-            'test-mac-snowleopard': 'snowleopard',
-            'test-linux-x86_64': 'lucid',
-        }
-        self._version = version_map[self._name]
+        self._operating_system = self._name.split('-')[1]
+        if self._operating_system == 'linux':
+            self._os_version = None
+        else:
+            self._os_version = VersionNameMap.map(self.host.platform).from_name(self._name.split('-')[2])[1]
+
+    def version_name(self):
+        if self._os_version is None:
+            return None
+        return VersionNameMap.map(self.host.platform).to_name(self._os_version, platform=self._operating_system, table=PUBLIC_TABLE)
 
     def default_pixel_tests(self):
         return True
@@ -397,10 +395,10 @@ class TestPort(Port):
         search_paths = {
             'test-mac-snowleopard': ['test-mac-snowleopard'],
             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
-            'test-win-win7': ['test-win-win7'],
-            'test-win-vista': ['test-win-vista', 'test-win-win7'],
-            'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
-            'test-linux-x86_64': ['test-linux', 'test-win-win7'],
+            'test-win-7sp0': ['test-win-7sp0'],
+            'test-win-vista': ['test-win-vista', 'test-win-7sp0'],
+            'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-7sp0'],
+            'test-linux-x86_64': ['test-linux'],
         }
         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
 
@@ -410,10 +408,10 @@ class TestPort(Port):
     def worker_startup_delay_secs(self):
         return 0
 
-    def check_build(self, needs_http):
+    def check_build(self):
         return True
 
-    def check_sys_deps(self, needs_http):
+    def check_sys_deps(self):
         return True
 
     def default_configuration(self):
@@ -442,8 +440,7 @@ class TestPort(Port):
 
     def _skipped_tests_for_unsupported_features(self, test_list):
         return set(['failures/expected/skip_text.html',
-                    'failures/unexpected/skip_pass.html',
-                    'virtual/skipped'])
+                    'failures/unexpected/skip_pass.html'])
 
     def name(self):
         return self._name
@@ -454,19 +451,19 @@ class TestPort(Port):
     def default_results_directory(self):
         return '/tmp/layout-test-results'
 
-    def setup_test_run(self):
+    def setup_test_run(self, device_class=None):
         pass
 
     def _driver_class(self):
         return TestDriver
 
-    def start_http_server(self, additional_dirs=None, number_of_servers=None):
-        pass
+    def path_to_crash_logs(self):
+        return self.results_directory()
 
-    def start_websocket_server(self):
+    def start_http_server(self, additional_dirs=None):
         pass
 
-    def acquire_http_lock(self):
+    def start_websocket_server(self):
         pass
 
     def stop_http_server(self):
@@ -475,9 +472,6 @@ class TestPort(Port):
     def stop_websocket_server(self):
         pass
 
-    def release_http_lock(self):
-        pass
-
     def _path_to_lighttpd(self):
         return "/usr/sbin/lighttpd"
 
@@ -514,7 +508,7 @@ class TestPort(Port):
                 ('snowleopard', 'x86'),
                 ('xp', 'x86'),
                 ('vista', 'x86'),
-                ('win7', 'x86'),
+                ('7sp0', 'x86'),
                 ('lucid', 'x86'),
                 ('lucid', 'x86_64'))
 
@@ -523,17 +517,11 @@ class TestPort(Port):
 
     def configuration_specifier_macros(self):
         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
-        return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
+        return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', '7sp0'], 'linux': ['lucid']}
 
     def all_baseline_variants(self):
         return self.ALL_BASELINE_VARIANTS
 
-    def virtual_test_suites(self):
-        return [
-            VirtualTestSuite('virtual/passes', 'passes', ['--virtual-arg']),
-            VirtualTestSuite('virtual/skipped', 'failures/expected', ['--virtual-arg2']),
-        ]
-
 
 class TestDriver(Driver):
     """Test/Dummy implementation of the DumpRenderTree interface."""
@@ -576,7 +564,7 @@ class TestDriver(Driver):
             actual_text = actual_text + ' ' + ' '.join(test_args)
 
         if test.actual_audio:
-            audio = base64.b64decode(test.actual_audio)
+            audio = test.actual_audio
         crashed_process_name = None
         crashed_pid = None
         if test.crash:
@@ -588,7 +576,7 @@ class TestDriver(Driver):
 
         crash_log = ''
         if crashed_process_name:
-            crash_logs = CrashLogs(self._port.host)
+            crash_logs = CrashLogs(self._port.host, self._port.path_to_crash_logs())
             crash_log = crash_logs.find_newest_log(crashed_process_name, None) or ''
 
         if stop_when_done: