run-apit-tests: Add --iterations and --repeat-each
authorjbedard@apple.com <jbedard@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 4 May 2019 01:03:20 +0000 (01:03 +0000)
committerjbedard@apple.com <jbedard@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 4 May 2019 01:03:20 +0000 (01:03 +0000)
https://bugs.webkit.org/show_bug.cgi?id=197563

Reviewed by Aakash Jain.

* Scripts/webkitpy/api_tests/manager.py:
(Manager.run): Add --iterations and --repeat-each/
* Scripts/webkitpy/api_tests/run_api_tests.py:
(parse_args): Duplicate the test list based on --iterations and --repeat-each.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@244942 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/api_tests/manager.py
Tools/Scripts/webkitpy/api_tests/run_api_tests.py

index 693628a..012e681 100644 (file)
@@ -1,3 +1,15 @@
+2019-05-03  Jonathan Bedard  <jbedard@apple.com>
+
+        run-apit-tests: Add --iterations and --repeat-each
+        https://bugs.webkit.org/show_bug.cgi?id=197563
+
+        Reviewed by Aakash Jain.
+
+        * Scripts/webkitpy/api_tests/manager.py:
+        (Manager.run): Add --iterations and --repeat-each/
+        * Scripts/webkitpy/api_tests/run_api_tests.py:
+        (parse_args): Duplicate the test list based on --iterations and --repeat-each.
+
 2019-05-03  Aakash Jain  <aakash_jain@apple.com>
 
         New EWS: Clicking on white bubble navigates to page with only bubbles
index 3237e92..f67add8 100644 (file)
@@ -182,10 +182,16 @@ class Manager(object):
                 self._stream.writeln(test)
             return Manager.SUCCESS
 
+        test_names = [test for test in test_names for _ in xrange(self._options.repeat_each)]
+        if self._options.repeat_each != 1:
+            _log.debug('Repeating each test {} times'.format(self._options.iterations))
+
         try:
             _log.info('Running tests')
             runner = Runner(self._port, self._stream)
-            runner.run(test_names, int(self._options.child_processes) if self._options.child_processes else self._port.default_child_processes())
+            for i in xrange(self._options.iterations):
+                _log.debug('\nIteration {}'.format(i + 1))
+                runner.run(test_names, int(self._options.child_processes) if self._options.child_processes else self._port.default_child_processes())
         except KeyboardInterrupt:
             # If we receive a KeyboardInterrupt, print results.
             self._stream.writeln('')
@@ -205,7 +211,7 @@ class Manager(object):
 
         self._stream.writeln('-' * 30)
         result = Manager.SUCCESS
-        if len(successful) + disabled == len(test_names):
+        if len(successful) * self._options.repeat_each + disabled == len(test_names):
             self._stream.writeln('All tests successfully passed!')
             if json_output:
                 self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
index c0fc7b0..31e21cf 100644 (file)
@@ -123,6 +123,8 @@ def parse_args(args):
                              help='Number of seconds to wait before a test times out'),
         optparse.make_option('--no-timeout', dest='timeout', action='store_false',
                              help='Disable timeouts for all tests'),
+        optparse.make_option('--iterations', type='int', default=1, help='Number of times to run the set of tests (e.g. ABCABCABC)'),
+        optparse.make_option('--repeat-each', type='int', default=1, help='Number of times to run each test (e.g. AAABBBCCC)'),
 
         # FIXME: Remove the default, API tests should be multiprocess
         optparse.make_option('--child-processes', default=1,