Some perf tests time out when ran by run-perf-tests
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests_integrationtest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 # Copyright (C) 2011 Apple Inc. All rights reserved.
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions are
8 # met:
9 #
10 #     * Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 #     * Redistributions in binary form must reproduce the above
13 # copyright notice, this list of conditions and the following disclaimer
14 # in the documentation and/or other materials provided with the
15 # distribution.
16 #     * Neither the name of Google Inc. nor the names of its
17 # contributors may be used to endorse or promote products derived from
18 # this software without specific prior written permission.
19 #
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32 """Unit tests for run_webkit_tests."""
33
34 from __future__ import with_statement
35
36 import codecs
37 import itertools
38 import logging
39 import Queue
40 import re
41 import sys
42 import thread
43 import time
44 import threading
45 import unittest
46
47 from webkitpy.common.system import path
48
49 try:
50     import multiprocessing
51 except ImportError:
52     multiprocessing = None
53
54 try:
55     import json
56 except ImportError:
57     # python 2.5 compatibility
58     import webkitpy.thirdparty.simplejson as json
59
60 # FIXME: remove this when we fix test-webkitpy to work properly on cygwin
61 # (bug 63846).
62 SHOULD_TEST_PROCESSES = multiprocessing and sys.platform not in ('cygwin', 'win32')
63
64 from webkitpy.common import array_stream
65 from webkitpy.common.system import outputcapture
66 from webkitpy.common.system.crashlogs_unittest import make_mock_crash_report_darwin
67 from webkitpy.common.host_mock import MockHost
68
69 from webkitpy.layout_tests import port
70 from webkitpy.layout_tests import run_webkit_tests
71 from webkitpy.layout_tests.port import Port
72 from webkitpy.layout_tests.port.test import TestPort, TestDriver
73 from webkitpy.test.skip import skip_if
74
75
76 def parse_args(extra_args=None, record_results=False, tests_included=False, new_results=False, print_nothing=True):
77     extra_args = extra_args or []
78     if print_nothing:
79         args = ['--print', 'nothing']
80     else:
81         args = []
82     if not '--platform' in extra_args:
83         args.extend(['--platform', 'test'])
84     if not record_results:
85         args.append('--no-record-results')
86     if not new_results:
87         args.append('--no-new-test-results')
88
89     if not '--child-processes' in extra_args and not '--worker-model' in extra_args:
90         args.extend(['--worker-model', 'inline'])
91     args.extend(extra_args)
92     if not tests_included:
93         # We use the glob to test that globbing works.
94         args.extend(['passes',
95                      'http/tests',
96                      'websocket/tests',
97                      'failures/expected/*'])
98     return run_webkit_tests.parse_args(args)
99
100
101 def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None):
102     options, parsed_args = parse_args(extra_args, record_results, tests_included)
103     if not port_obj:
104         host = host or MockHost()
105         port_obj = host.port_factory.get(port_name=options.platform, options=options)
106     buildbot_output = array_stream.ArrayStream()
107     regular_output = array_stream.ArrayStream()
108     res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output, regular_output=regular_output)
109     return res == 0 and regular_output.empty() and buildbot_output.empty()
110
111
112 def logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, host=None, new_results=False):
113     options, parsed_args = parse_args(extra_args=extra_args,
114                                       record_results=record_results,
115                                       tests_included=tests_included,
116                                       print_nothing=False, new_results=new_results)
117     host = host or MockHost()
118     if not port_obj:
119         port_obj = host.port_factory.get(port_name=options.platform, options=options)
120
121     res, buildbot_output, regular_output = run_and_capture(port_obj, options, parsed_args)
122     return (res, buildbot_output, regular_output, host.user)
123
124
125 def run_and_capture(port_obj, options, parsed_args):
126     oc = outputcapture.OutputCapture()
127     try:
128         oc.capture_output()
129         buildbot_output = array_stream.ArrayStream()
130         regular_output = array_stream.ArrayStream()
131         res = run_webkit_tests.run(port_obj, options, parsed_args,
132                                    buildbot_output=buildbot_output,
133                                    regular_output=regular_output)
134     finally:
135         oc.restore_output()
136     return (res, buildbot_output, regular_output)
137
138
139 def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
140                   host=None, include_reference_html=False):
141     extra_args = extra_args or []
142     if not tests_included:
143         # Not including http tests since they get run out of order (that
144         # behavior has its own test, see test_get_test_file_queue)
145         extra_args = ['passes', 'failures'] + extra_args
146     options, parsed_args = parse_args(extra_args, tests_included=True)
147
148     host = host or MockHost()
149     test_batches = []
150
151     class RecordingTestDriver(TestDriver):
152         def __init__(self, port, worker_number):
153             TestDriver.__init__(self, port, worker_number, pixel_tests=port.get_option('pixel_test'), no_timeout=False)
154             self._current_test_batch = None
155
156         def stop(self):
157             self._current_test_batch = None
158
159         def run_test(self, test_input):
160             if self._current_test_batch is None:
161                 self._current_test_batch = []
162                 test_batches.append(self._current_test_batch)
163             test_name = test_input.test_name
164             # In case of reftest, one test calls the driver's run_test() twice.
165             # We should not add a reference html used by reftests to tests unless include_reference_html parameter
166             # is explicitly given.
167             filesystem = self._port.host.filesystem
168             dirname, filename = filesystem.split(test_name)
169             if include_reference_html or not Port.is_reference_html_file(filesystem, dirname, filename):
170                 self._current_test_batch.append(test_name)
171             return TestDriver.run_test(self, test_input)
172
173     class RecordingTestPort(TestPort):
174         def create_driver(self, worker_number):
175             return RecordingTestDriver(self, worker_number)
176
177     recording_port = RecordingTestPort(host, options=options)
178     run_and_capture(recording_port, options, parsed_args)
179
180     if flatten_batches:
181         return list(itertools.chain(*test_batches))
182
183     return test_batches
184
185
186 # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
187 # FIXME: It's nice to have a routine in port/test.py that returns this number.
188 unexpected_tests_count = 12
189
190
191 class MainTest(unittest.TestCase):
192     def test_accelerated_compositing(self):
193         # This just tests that we recognize the command line args
194         self.assertTrue(passing_run(['--accelerated-video']))
195         self.assertTrue(passing_run(['--no-accelerated-video']))
196
197     def test_accelerated_2d_canvas(self):
198         # This just tests that we recognize the command line args
199         self.assertTrue(passing_run(['--accelerated-2d-canvas']))
200         self.assertTrue(passing_run(['--no-accelerated-2d-canvas']))
201
202     def test_basic(self):
203         self.assertTrue(passing_run())
204
205     def test_batch_size(self):
206         batch_tests_run = get_tests_run(['--batch-size', '2'])
207         for batch in batch_tests_run:
208             self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
209
210     def test_child_process_1(self):
211         if SHOULD_TEST_PROCESSES:
212             _, _, regular_output, _ = logging_run(
213                 ['--print', 'config', '--worker-model', 'processes', '--child-processes', '1'])
214             self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()]))
215
216     def test_child_processes_2(self):
217         # This test seems to fail on win32.
218         if sys.platform == 'win32':
219             return
220         if SHOULD_TEST_PROCESSES:
221             _, _, regular_output, _ = logging_run(
222                 ['--print', 'config', '--worker-model', 'processes', '--child-processes', '2'])
223             self.assertTrue(any(['Running 2 ' in line for line in regular_output.get()]))
224
225     def test_child_processes_min(self):
226         if SHOULD_TEST_PROCESSES:
227             _, _, regular_output, _ = logging_run(
228                 ['--print', 'config', '--worker-model', 'processes', '--child-processes', '2', 'passes'],
229                 tests_included=True)
230             self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()]))
231
232     def test_dryrun(self):
233         batch_tests_run = get_tests_run(['--dry-run'])
234         self.assertEqual(batch_tests_run, [])
235
236         batch_tests_run = get_tests_run(['-n'])
237         self.assertEqual(batch_tests_run, [])
238
239     def test_exception_raised(self):
240         # Exceptions raised by a worker are treated differently depending on
241         # whether they are in-process or out. inline exceptions work as normal,
242         # which allows us to get the full stack trace and traceback from the
243         # worker. The downside to this is that it could be any error, but this
244         # is actually useful in testing, which is what --worker-model=inline is
245         # usually used for.
246         #
247         # Exceptions raised in a separate process are re-packaged into
248         # WorkerExceptions, which have a string capture of the stack which can
249         # be printed, but don't display properly in the unit test exception handlers.
250         self.assertRaises(ValueError, logging_run,
251             ['failures/expected/exception.html'], tests_included=True)
252
253         if SHOULD_TEST_PROCESSES:
254             self.assertRaises(run_webkit_tests.WorkerException, logging_run,
255                 ['--worker-model', 'processes', 'failures/expected/exception.html'], tests_included=True)
256
257     def test_full_results_html(self):
258         # FIXME: verify html?
259         res, out, err, user = logging_run(['--full-results-html'])
260         self.assertEqual(res, 0)
261
262     def test_help_printing(self):
263         res, out, err, user = logging_run(['--help-printing'])
264         self.assertEqual(res, 0)
265         self.assertTrue(out.empty())
266         self.assertFalse(err.empty())
267
268     def test_hung_thread(self):
269         res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50',
270                                           'failures/expected/hang.html'],
271                                           tests_included=True)
272         self.assertEqual(res, 0)
273         self.assertFalse(out.empty())
274         self.assertFalse(err.empty())
275
276     def test_keyboard_interrupt(self):
277         # Note that this also tests running a test marked as SKIP if
278         # you specify it explicitly.
279         self.assertRaises(KeyboardInterrupt, logging_run,
280             ['failures/expected/keyboard.html'], tests_included=True)
281
282     def test_keyboard_interrupt_inline_worker_model(self):
283         self.assertRaises(KeyboardInterrupt, logging_run,
284             ['failures/expected/keyboard.html', '--worker-model', 'inline'],
285             tests_included=True)
286
287     def test_lint_test_files(self):
288         res, out, err, user = logging_run(['--lint-test-files'])
289         self.assertEqual(res, 0)
290         self.assertTrue(out.empty())
291         self.assertTrue(any(['Lint succeeded' in msg for msg in err.get()]))
292
293     def test_lint_test_files__errors(self):
294         options, parsed_args = parse_args(['--lint-test-files'])
295         host = MockHost()
296         port_obj = host.port_factory.get(options.platform, options=options)
297         port_obj.test_expectations = lambda: "# syntax error"
298         res, out, err = run_and_capture(port_obj, options, parsed_args)
299
300         self.assertEqual(res, -1)
301         self.assertTrue(out.empty())
302         self.assertTrue(any(['Lint failed' in msg for msg in err.get()]))
303
304     def test_no_tests_found(self):
305         res, out, err, user = logging_run(['resources'], tests_included=True)
306         self.assertEqual(res, -1)
307         self.assertTrue(out.empty())
308         self.assertTrue('No tests to run.\n' in err.get())
309
310     def test_no_tests_found_2(self):
311         res, out, err, user = logging_run(['foo'], tests_included=True)
312         self.assertEqual(res, -1)
313         self.assertTrue(out.empty())
314         self.assertTrue('No tests to run.\n' in err.get())
315
316     def test_randomize_order(self):
317         # FIXME: verify order was shuffled
318         self.assertTrue(passing_run(['--randomize-order']))
319
320     def test_gc_between_tests(self):
321         self.assertTrue(passing_run(['--gc-between-tests']))
322
323     def test_complex_text(self):
324         self.assertTrue(passing_run(['--complex-text']))
325
326     def test_threaded(self):
327         self.assertTrue(passing_run(['--threaded']))
328
329     def test_repeat_each(self):
330         tests_to_run = ['passes/image.html', 'passes/text.html']
331         tests_run = get_tests_run(['--repeat-each', '2'] + tests_to_run, tests_included=True, flatten_batches=True)
332         self.assertEquals(tests_run, ['passes/image.html', 'passes/image.html', 'passes/text.html', 'passes/text.html'])
333
334     def test_iterations(self):
335         tests_to_run = ['passes/image.html', 'passes/text.html']
336         tests_run = get_tests_run(['--iterations', '2'] + tests_to_run, tests_included=True, flatten_batches=True)
337         self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html', 'passes/image.html', 'passes/text.html'])
338
339     def test_run_chunk(self):
340         # Test that we actually select the right chunk
341         all_tests_run = get_tests_run(flatten_batches=True)
342         chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True)
343         self.assertEquals(all_tests_run[4:8], chunk_tests_run)
344
345         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
346         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
347         chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True)
348         self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
349
350     def test_run_force(self):
351         # This raises an exception because we run
352         # failures/expected/exception.html, which is normally SKIPped.
353
354         # See also the comments in test_exception_raised() about ValueError vs. WorkerException.
355         self.assertRaises(ValueError, logging_run, ['--force'])
356
357     def test_run_part(self):
358         # Test that we actually select the right part
359         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
360         tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True)
361         self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run)
362
363         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
364         # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
365         # last part repeats the first two tests).
366         chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True)
367         self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run)
368
369     def test_run_singly(self):
370         batch_tests_run = get_tests_run(['--run-singly'])
371         for batch in batch_tests_run:
372             self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
373
374     def test_skip_failing_tests(self):
375         batches = get_tests_run(['--skip-failing-tests'])
376         has_passes_text = False
377         for batch in batches:
378             self.assertFalse('failures/expected/text.html' in batch)
379             has_passes_text = has_passes_text or ('passes/text.html' in batch)
380         self.assertTrue(has_passes_text)
381
382     def test_run_singly_actually_runs_tests(self):
383         res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
384         self.assertEquals(res, 8)
385
386     def test_single_file(self):
387         # FIXME: We should consider replacing more of the get_tests_run()-style tests
388         # with tests that read the tests_run* files, like this one.
389         host = MockHost()
390         tests_run = passing_run(['passes/text.html'], tests_included=True, host=host)
391         self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/tests_run0.txt'),
392                           'passes/text.html\n')
393
394     def test_single_file_with_prefix(self):
395         tests_run = get_tests_run(['LayoutTests/passes/text.html'], tests_included=True, flatten_batches=True)
396         self.assertEquals(['passes/text.html'], tests_run)
397
398     def test_single_skipped_file(self):
399         tests_run = get_tests_run(['failures/expected/keybaord.html'], tests_included=True, flatten_batches=True)
400         self.assertEquals([], tests_run)
401
402     def test_stderr_is_saved(self):
403         host = MockHost()
404         self.assertTrue(passing_run(host=host))
405         self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
406                           'stuff going to stderr')
407
408     def test_test_list(self):
409         host = MockHost()
410         filename = '/tmp/foo.txt'
411         host.filesystem.write_text_file(filename, 'passes/text.html')
412         tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, host=host)
413         self.assertEquals(['passes/text.html'], tests_run)
414         host.filesystem.remove(filename)
415         res, out, err, user = logging_run(['--test-list=%s' % filename],
416                                           tests_included=True, host=host)
417         self.assertEqual(res, -1)
418         self.assertFalse(err.empty())
419
420     def test_test_list_with_prefix(self):
421         host = MockHost()
422         filename = '/tmp/foo.txt'
423         host.filesystem.write_text_file(filename, 'LayoutTests/passes/text.html')
424         tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, host=host)
425         self.assertEquals(['passes/text.html'], tests_run)
426
427     def test_unexpected_failures(self):
428         # Run tests including the unexpected failures.
429         self._url_opened = None
430         res, out, err, user = logging_run(tests_included=True)
431
432         self.assertEqual(res, unexpected_tests_count)
433         self.assertFalse(out.empty())
434         self.assertFalse(err.empty())
435         self.assertEqual(user.opened_urls, [path.abspath_to_uri('/tmp/layout-test-results/results.html')])
436
437     def test_missing_and_unexpected_results(self):
438         # Test that we update expectations in place. If the expectation
439         # is missing, update the expected generic location.
440         host = MockHost()
441         res, out, err, _ = logging_run(['--no-show-results',
442             'failures/expected/missing_image.html',
443             'failures/unexpected/missing_text.html',
444             'failures/unexpected/text-image-checksum.html'],
445             tests_included=True, host=host, record_results=True)
446         file_list = host.filesystem.written_files.keys()
447         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
448         self.assertEquals(res, 1)
449         expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
450         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
451         self.assertTrue(json_string.find(expected_token) != -1)
452         self.assertTrue(json_string.find('"num_regressions":1') != -1)
453         self.assertTrue(json_string.find('"num_flaky":0') != -1)
454         self.assertTrue(json_string.find('"num_missing":1') != -1)
455
456     def test_missing_and_unexpected_results_with_custom_exit_code(self):
457         # Test that we update expectations in place. If the expectation
458         # is missing, update the expected generic location.
459         class CustomExitCodePort(TestPort):
460             def exit_code_from_summarized_results(self, unexpected_results):
461                 return unexpected_results['num_regressions'] + unexpected_results['num_missing']
462
463         host = MockHost()
464         options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
465         test_port = CustomExitCodePort(host, options=options)
466         res, out, err, _ = logging_run(['--no-show-results',
467             'failures/expected/missing_image.html',
468             'failures/unexpected/missing_text.html',
469             'failures/unexpected/text-image-checksum.html'],
470             tests_included=True, host=host, record_results=True, port_obj=test_port)
471         self.assertEquals(res, 2)
472
473     def test_crash_with_stderr(self):
474         host = MockHost()
475         res, buildbot_output, regular_output, user = logging_run([
476                 'failures/unexpected/crash-with-stderr.html',
477             ],
478             tests_included=True,
479             record_results=True,
480             host=host)
481         self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
482
483     def test_no_image_failure_with_image_diff(self):
484         host = MockHost()
485         res, buildbot_output, regular_output, user = logging_run([
486                 'failures/unexpected/checksum-with-matching-image.html',
487             ],
488             tests_included=True,
489             record_results=True,
490             host=host)
491         self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1)
492
493     def test_crash_log(self):
494         mock_crash_report = make_mock_crash_report_darwin('DumpRenderTree', 12345)
495         host = MockHost()
496         host.filesystem.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/DumpRenderTree_2011-06-13-150719_quadzen.crash', mock_crash_report)
497         res, buildbot_output, regular_output, user = logging_run([
498                 'failures/unexpected/crash-with-stderr.html',
499             ],
500             tests_included=True,
501             record_results=True,
502             host=host)
503         expected_crash_log = mock_crash_report
504         # Currently CrashLog uploading only works on Darwin.
505         if sys.platform != "darwin":
506             expected_crash_log = "mock-std-error-output"
507         self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/crash-with-stderr-crash-log.txt'), expected_crash_log)
508
509     def test_web_process_crash_log(self):
510         mock_crash_report = make_mock_crash_report_darwin('WebProcess', 12345)
511         host = MockHost()
512         host.filesystem.write_text_file('/Users/mock/Library/Logs/DiagnosticReports/WebProcess_2011-06-13-150719_quadzen.crash', mock_crash_report)
513         res, buildbot_output, regular_output, user = logging_run([
514                 'failures/unexpected/web-process-crash-with-stderr.html',
515             ],
516             tests_included=True,
517             record_results=True,
518             host=host)
519         expected_crash_log = mock_crash_report
520         # Currently CrashLog uploading only works on Darwin.
521         if sys.platform != "darwin":
522             expected_crash_log = "mock-std-error-output"
523         self.assertEquals(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/web-process-crash-with-stderr-crash-log.txt'), expected_crash_log)
524
525     def test_exit_after_n_failures_upload(self):
526         host = MockHost()
527         res, buildbot_output, regular_output, user = logging_run([
528                 'failures/unexpected/text-image-checksum.html',
529                 'passes/text.html',
530                 '--exit-after-n-failures', '1',
531             ],
532             tests_included=True,
533             record_results=True,
534             host=host)
535         self.assertTrue('/tmp/layout-test-results/incremental_results.json' in host.filesystem.files)
536
537     def test_exit_after_n_failures(self):
538         # Unexpected failures should result in tests stopping.
539         tests_run = get_tests_run([
540                 'failures/unexpected/text-image-checksum.html',
541                 'passes/text.html',
542                 '--exit-after-n-failures', '1',
543             ],
544             tests_included=True,
545             flatten_batches=True)
546         self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run)
547
548         # But we'll keep going for expected ones.
549         tests_run = get_tests_run([
550                 'failures/expected/text.html',
551                 'passes/text.html',
552                 '--exit-after-n-failures', '1',
553             ],
554             tests_included=True,
555             flatten_batches=True)
556         self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run)
557
558     def test_exit_after_n_crashes(self):
559         # Unexpected crashes should result in tests stopping.
560         tests_run = get_tests_run([
561                 'failures/unexpected/crash.html',
562                 'passes/text.html',
563                 '--exit-after-n-crashes-or-timeouts', '1',
564             ],
565             tests_included=True,
566             flatten_batches=True)
567         self.assertEquals(['failures/unexpected/crash.html'], tests_run)
568
569         # Same with timeouts.
570         tests_run = get_tests_run([
571                 'failures/unexpected/timeout.html',
572                 'passes/text.html',
573                 '--exit-after-n-crashes-or-timeouts', '1',
574             ],
575             tests_included=True,
576             flatten_batches=True)
577         self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
578
579         # But we'll keep going for expected ones.
580         tests_run = get_tests_run([
581                 'failures/expected/crash.html',
582                 'passes/text.html',
583                 '--exit-after-n-crashes-or-timeouts', '1',
584             ],
585             tests_included=True,
586             flatten_batches=True)
587         self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run)
588
589     def test_exit_after_n_crashes_inline_worker_model(self):
590         tests_run = get_tests_run([
591                 'failures/unexpected/timeout.html',
592                 'passes/text.html',
593                 '--exit-after-n-crashes-or-timeouts', '1',
594                 '--worker-model', 'inline',
595             ],
596             tests_included=True,
597             flatten_batches=True)
598         self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
599
600     def test_results_directory_absolute(self):
601         # We run a configuration that should fail, to generate output, then
602         # look for what the output results url was.
603
604         host = MockHost()
605         with host.filesystem.mkdtemp() as tmpdir:
606             res, out, err, user = logging_run(['--results-directory=' + str(tmpdir)],
607                                               tests_included=True, host=host)
608             self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.filesystem.join(tmpdir, 'results.html'))])
609
610     def test_results_directory_default(self):
611         # We run a configuration that should fail, to generate output, then
612         # look for what the output results url was.
613
614         # This is the default location.
615         res, out, err, user = logging_run(tests_included=True)
616         self.assertEqual(user.opened_urls, [path.abspath_to_uri('/tmp/layout-test-results/results.html')])
617
618     def test_results_directory_relative(self):
619         # We run a configuration that should fail, to generate output, then
620         # look for what the output results url was.
621         host = MockHost()
622         host.filesystem.maybe_make_directory('/tmp/cwd')
623         host.filesystem.chdir('/tmp/cwd')
624         res, out, err, user = logging_run(['--results-directory=foo'],
625                                           tests_included=True, host=host)
626         self.assertEqual(user.opened_urls, [path.abspath_to_uri('/tmp/cwd/foo/results.html')])
627
628     def test_retries_directory(self):
629         host = MockHost()
630         res, out, err, user = logging_run(tests_included=True, host=host)
631         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/tests_run0.txt'))
632
633     # These next tests test that we run the tests in ascending alphabetical
634     # order per directory. HTTP tests are sharded separately from other tests,
635     # so we have to test both.
636     def assert_run_order(self, worker_model, child_processes='1'):
637         tests_run = get_tests_run(['--worker-model', worker_model,
638             '--child-processes', child_processes, 'passes'],
639             tests_included=True, flatten_batches=True)
640         self.assertEquals(tests_run, sorted(tests_run))
641
642         tests_run = get_tests_run(['--worker-model', worker_model,
643             '--child-processes', child_processes, 'http/tests/passes'],
644             tests_included=True, flatten_batches=True)
645         self.assertEquals(tests_run, sorted(tests_run))
646
647     def test_run_order__inline(self):
648         self.assert_run_order('inline')
649
650     def test_tolerance(self):
651         class ImageDiffTestPort(TestPort):
652             def diff_image(self, expected_contents, actual_contents, tolerance=None):
653                 self.tolerance_used_for_diff_image = self._options.tolerance
654                 return (True, 1)
655
656         def get_port_for_run(args):
657             options, parsed_args = run_webkit_tests.parse_args(args)
658             host = MockHost()
659             test_port = ImageDiffTestPort(host, options=options)
660             res = passing_run(args, port_obj=test_port, tests_included=True)
661             self.assertTrue(res)
662             return test_port
663
664         base_args = ['--pixel-tests', '--no-new-test-results', 'failures/expected/*']
665
666         # If we pass in an explicit tolerance argument, then that will be used.
667         test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
668         self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
669         test_port = get_port_for_run(base_args + ['--tolerance', '0'])
670         self.assertEqual(0, test_port.tolerance_used_for_diff_image)
671
672         # Otherwise the port's default tolerance behavior (including ignoring it)
673         # should be used.
674         test_port = get_port_for_run(base_args)
675         self.assertEqual(None, test_port.tolerance_used_for_diff_image)
676
677     def test_worker_model__inline(self):
678         self.assertTrue(passing_run(['--worker-model', 'inline']))
679
680     def test_worker_model__inline_with_child_processes(self):
681         res, out, err, user = logging_run(['--worker-model', 'inline',
682                                            '--child-processes', '2'])
683         self.assertEqual(res, 0)
684         self.assertTrue('--worker-model=inline overrides --child-processes\n' in err.get())
685
686     def test_worker_model__processes(self):
687         if SHOULD_TEST_PROCESSES:
688             self.assertTrue(passing_run(['--worker-model', 'processes']))
689
690     def test_worker_model__processes_and_dry_run(self):
691         if SHOULD_TEST_PROCESSES:
692             self.assertTrue(passing_run(['--worker-model', 'processes', '--dry-run']))
693
694     def test_worker_model__unknown(self):
695         self.assertRaises(ValueError, logging_run, ['--worker-model', 'unknown'])
696
697     def test_reftest_run(self):
698         tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True)
699         self.assertEquals(['passes/reftest.html'], tests_run)
700
701     def test_reftest_run_reftests_if_pixel_tests_are_disabled(self):
702         tests_run = get_tests_run(['--no-pixel-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
703         self.assertEquals(['passes/reftest.html'], tests_run)
704
705     def test_reftest_skip_reftests_if_no_ref_tests(self):
706         tests_run = get_tests_run(['--no-ref-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
707         self.assertEquals([], tests_run)
708         tests_run = get_tests_run(['--no-ref-tests', '--no-pixel-tests', 'passes/reftest.html'], tests_included=True, flatten_batches=True)
709         self.assertEquals([], tests_run)
710
711     def test_reftest_expected_html_should_be_ignored(self):
712         tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True)
713         self.assertEquals([], tests_run)
714
715     def test_reftest_driver_should_run_expected_html(self):
716         tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True, include_reference_html=True)
717         self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run)
718
719     def test_reftest_driver_should_run_expected_mismatch_html(self):
720         tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True, include_reference_html=True)
721         self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run)
722
723     def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
724         host = MockHost()
725         res, out, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host, record_results=True)
726         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
727         self.assertTrue(json_string.find('"unlistedtest.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
728         self.assertTrue(json_string.find('"num_regressions":4') != -1)
729         self.assertTrue(json_string.find('"num_flaky":0') != -1)
730         self.assertTrue(json_string.find('"num_missing":1') != -1)
731
732     def test_additional_platform_directory(self):
733         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
734         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
735         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', '--additional-platform-directory', '/tmp/bar']))
736
737         res, buildbot_output, regular_output, user = logging_run(['--additional-platform-directory', 'foo'])
738         self.assertTrue('--additional-platform-directory=foo is ignored since it is not absolute\n' in regular_output.get())
739
740     def test_no_http_and_force(self):
741         # See test_run_force, using --force raises an exception.
742         # FIXME: We would like to check the warnings generated.
743         self.assertRaises(ValueError, logging_run, ['--force', '--no-http'])
744
745     @staticmethod
746     def has_test_of_type(tests, type):
747         return [test for test in tests if type in test]
748
749     def test_no_http_tests(self):
750         batch_tests_dryrun = get_tests_run(['LayoutTests/http', 'websocket/'], flatten_batches=True)
751         self.assertTrue(MainTest.has_test_of_type(batch_tests_dryrun, 'http'))
752         self.assertTrue(MainTest.has_test_of_type(batch_tests_dryrun, 'websocket'))
753
754         batch_tests_run_no_http = get_tests_run(['--no-http', 'LayoutTests/http', 'websocket/'], flatten_batches=True)
755         self.assertFalse(MainTest.has_test_of_type(batch_tests_run_no_http, 'http'))
756         self.assertFalse(MainTest.has_test_of_type(batch_tests_run_no_http, 'websocket'))
757
758         batch_tests_run_http = get_tests_run(['--http', 'LayoutTests/http', 'websocket/'], flatten_batches=True)
759         self.assertTrue(MainTest.has_test_of_type(batch_tests_run_http, 'http'))
760         self.assertTrue(MainTest.has_test_of_type(batch_tests_run_http, 'websocket'))
761
762 MainTest = skip_if(MainTest, sys.platform == 'cygwin' and sys.version < '2.6', 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
763
764
765 class EndToEndTest(unittest.TestCase):
766     def parse_full_results(self, full_results_text):
767         json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
768         compressed_results = json.loads(json_to_eval)
769         return compressed_results
770
771     def test_end_to_end(self):
772         host = MockHost()
773         res, out, err, user = logging_run(record_results=True, tests_included=True, host=host)
774
775         self.assertEquals(res, unexpected_tests_count)
776         results = self.parse_full_results(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json'))
777
778         # Check to ensure we're passing back image diff %age correctly.
779         self.assertEquals(results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1)
780
781         # Check that we attempted to display the results page in a browser.
782         self.assertTrue(user.opened_urls)
783
784     def test_reftest_with_two_notrefs(self):
785         # Test that we update expectations in place. If the expectation
786         # is missing, update the expected generic location.
787         host = MockHost()
788         res, out, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host, record_results=True)
789         file_list = host.filesystem.written_files.keys()
790         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
791         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
792         json = self.parse_full_results(json_string)
793         self.assertTrue("multiple-match-success.html" not in json["tests"]["reftests"]["foo"])
794         self.assertTrue("multiple-mismatch-success.html" not in json["tests"]["reftests"]["foo"])
795         self.assertTrue("multiple-both-success.html" not in json["tests"]["reftests"]["foo"])
796         self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-match-failure.html"],
797             {"expected": "PASS", "ref_file": "reftests/foo/second-mismatching-ref.html", "actual": "IMAGE", 'is_reftest': True})
798         self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-mismatch-failure.html"],
799             {"expected": "PASS", "ref_file": "reftests/foo/matching-ref.html", "actual": "IMAGE", "is_mismatch_reftest": True})
800         self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-both-failure.html"],
801             {"expected": "PASS", "ref_file": "reftests/foo/matching-ref.html", "actual": "IMAGE", "is_mismatch_reftest": True})
802
803
804 class RebaselineTest(unittest.TestCase):
805     def assertBaselines(self, file_list, file, extensions, err):
806         "assert that the file_list contains the baselines."""
807         for ext in extensions:
808             baseline = file + "-expected" + ext
809             baseline_msg = 'Writing new expected result "%s"\n' % baseline[1:]
810             self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
811             self.assertTrue(baseline_msg in err.get())
812
813     # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
814     # supposed to be.
815
816     def test_reset_results(self):
817         # Test that we update expectations in place. If the expectation
818         # is missing, update the expected generic location.
819         host = MockHost()
820         res, out, err, _ = logging_run(['--pixel-tests',
821                         '--reset-results',
822                         'passes/image.html',
823                         'failures/expected/missing_image.html'],
824                         tests_included=True, host=host, new_results=True)
825         file_list = host.filesystem.written_files.keys()
826         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
827         self.assertEquals(res, 0)
828         self.assertTrue(out.empty())
829         self.assertEqual(len(file_list), 4)
830         self.assertBaselines(file_list, "/passes/image", [".txt", ".png"], err)
831         self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"], err)
832
833     def test_missing_results(self):
834         # Test that we update expectations in place. If the expectation
835         # is missing, update the expected generic location.
836         host = MockHost()
837         res, out, err, _ = logging_run(['--no-show-results',
838                      'failures/unexpected/missing_text.html',
839                      'failures/unexpected/missing_image.html',
840                      'failures/unexpected/missing_audio.html',
841                      'failures/unexpected/missing_render_tree_dump.html'],
842                      tests_included=True, host=host, new_results=True)
843         file_list = host.filesystem.written_files.keys()
844         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
845         self.assertEquals(res, 0)
846         self.assertFalse(out.empty())
847         self.assertEqual(len(file_list), 6)
848         self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"], err)
849         self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_image", [".png"], err)
850         self.assertBaselines(file_list, "/platform/test-mac-leopard/failures/unexpected/missing_render_tree_dump", [".txt"], err)
851
852     def test_new_baseline(self):
853         # Test that we update the platform expectations. If the expectation
854         # is mssing, then create a new expectation in the platform dir.
855         host = MockHost()
856         res, out, err, _ = logging_run(['--pixel-tests',
857                         '--new-baseline',
858                         'passes/image.html',
859                         'failures/expected/missing_image.html'],
860                     tests_included=True, host=host, new_results=True)
861         file_list = host.filesystem.written_files.keys()
862         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
863         self.assertEquals(res, 0)
864         self.assertTrue(out.empty())
865         self.assertEqual(len(file_list), 4)
866         self.assertBaselines(file_list,
867             "/platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
868         self.assertBaselines(file_list,
869             "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)
870
871
872 class DryrunTest(unittest.TestCase):
873     # FIXME: it's hard to know which platforms are safe to test; the
874     # chromium platforms require a chromium checkout, and the mac platform
875     # requires fcntl, so it can't be tested on win32, etc. There is
876     # probably a better way of handling this.
877     def disabled_test_darwin(self):
878         if sys.platform != "darwin":
879             return
880
881         self.assertTrue(passing_run(['--platform', 'dryrun', 'fast/html'], tests_included=True))
882         self.assertTrue(passing_run(['--platform', 'dryrun-mac', 'fast/html'], tests_included=True))
883
884     def test_test(self):
885         self.assertTrue(passing_run(['--platform', 'dryrun-test', '--pixel-tests']))
886
887
888 if __name__ == '__main__':
889     unittest.main()