nrwt should use multiple http shards

https://bugs.webkit.org/show_bug.cgi?id=75958

Add --max-locked-shards option to the nrwt for manual set
the maximum number of the locked shards

Reviewed by Dirk Pranke.

* Scripts/webkitpy/layout_tests/controllers/manager.py:
(Manager._max_locked_shards):
* Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:
(ShardingTests.get_shards):
(ShardingTests.test_multiple_locked_shards):
* Scripts/webkitpy/layout_tests/run_webkit_tests.py:
(parse_args):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@104967 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index 1ef73e3c45fe936063fd925924d047f1bcca3924..f805fb10b4e8760cba5ecf5a7de9d8054936aae5 100644 (file)
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,21 @@
+2012-01-13  Kristóf Kosztyó  <kkristof@inf.u-szeged.hu>
+
+        nrwt should use multiple http shards
+        https://bugs.webkit.org/show_bug.cgi?id=75958
+
+        Add --max-locked-shards option to the nrwt for manual set
+        the maximum number of the locked shards
+
+        Reviewed by Dirk Pranke.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (Manager._max_locked_shards):
+        * Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:
+        (ShardingTests.get_shards):
+        (ShardingTests.test_multiple_locked_shards):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        (parse_args):
+
 2012-01-13  Raphael Kubo da Costa  <kubo@profusion.mobi>
 
         prepare-ChangeLog: Look for a text editor in a few more variables.

diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
index aff6048336016550e756d537f885c83388f4e87d..76ccba549cbd780cc54dfbd9f7bfb69ebec8603c 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
@@ -668,11 +668,17 @@ class Manager(object):
         # Put a ceiling on the number of locked shards, so that we
         # don't hammer the servers too badly.
 
-        # FIXME: For now, limit to one shard. After testing to make sure we
+        # FIXME: For now, limit to one shard or set it
+        # with the --max-locked-shards. After testing to make sure we
         # can handle multiple shards, we should probably do something like
         # limit this to no more than a quarter of all workers, e.g.:
         # return max(math.ceil(num_workers / 4.0), 1)
-        return 1
+        if self._options.max_locked_shards:
+            num_of_locked_shards = self._options.max_locked_shards
+        else:
+            num_of_locked_shards = 1
+
+        return num_of_locked_shards
 
     def _resize_shards(self, old_shards, max_new_shards, shard_name_prefix):
         """Takes a list of shards and redistributes the tests into no more

diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py
index 98db2fc3ca84bdd788221fd899cda56951e771b3..1fb10def1d1089cba68d1b699e89f22d5974a301 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py
@@ -69,13 +69,13 @@ class ShardingTests(unittest.TestCase):
         "dom/html/level2/html/HTMLAnchorElement06.html",
     ]
 
-    def get_shards(self, num_workers, fully_parallel, test_list=None):
+    def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=None):
         test_list = test_list or self.test_list
         host = MockHost()
         port = host.port_factory.get(port_name='test')
         port._filesystem = MockFileSystem()
-        # FIXME: This should use MockOptions() instead of Mock()
-        self.manager = ManagerWrapper(port=port, options=Mock(), printer=Mock())
+        options = MockOptions(max_locked_shards=max_locked_shards)
+        self.manager = ManagerWrapper(port=port, options=options, printer=Mock())
         return self.manager._shard_tests(test_list, num_workers, fully_parallel)
 
     def test_shard_by_dir(self):
@@ -143,6 +143,24 @@ class ShardingTests(unittest.TestCase):
         self.assertEquals(len(locked), 1)
         self.assertEquals(len(unlocked), 0)
 
+    def test_multiple_locked_shards(self):
+        locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2)
+        self.assertEqual(locked,
+            [TestShard('locked_shard_1',
+                       ['http/tests/security/view-source-no-refresh.html',
+                        'http/tests/websocket/tests/unicode.htm',
+                        'http/tests/websocket/tests/websocket-protocol-ignored.html']),
+             TestShard('locked_shard_2',
+                        ['http/tests/xmlhttprequest/supported-xml-content-types.html'])])
+
+        locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False)
+        self.assertEquals(locked,
+            [TestShard('locked_shard_1',
+                       ['http/tests/security/view-source-no-refresh.html',
+                        'http/tests/websocket/tests/unicode.htm',
+                        'http/tests/websocket/tests/websocket-protocol-ignored.html',
+                        'http/tests/xmlhttprequest/supported-xml-content-types.html'])])
+
 
 class ManagerTest(unittest.TestCase):
     def get_options(self):

diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
index 9b485574857751b7e0137354482dbc1b34297c5d..b6068b6e027655e4c23ddb8e4583924812d2b98a 100755 (executable)
--- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
@@ -379,6 +379,8 @@ def parse_args(args=None):
         optparse.make_option("--no-retry-failures", action="store_false",
             dest="retry_failures",
             help="Don't re-try any tests that produce unexpected results."),
+        optparse.make_option("--max-locked-shards", type="int",
+            help="Set the maximum number of locked shards"),
     ]
 
     misc_options = [