Clean up ChunkedUpdateDrawingAreaProxy
[WebKit-https.git] / Tools / Scripts / webkitpy / common / net / buildbot / buildbot_unittest.py
1 # Copyright (C) 2009 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #    * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #    * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #    * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import unittest
30
31 from webkitpy.common.net.layouttestresults import LayoutTestResults
32 from webkitpy.common.net.buildbot import BuildBot, Builder, Build
33 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
34
35
36 class BuilderTest(unittest.TestCase):
37     def _install_fetch_build(self, failure):
38         def _mock_fetch_build(build_number):
39             build = Build(
40                 builder=self.builder,
41                 build_number=build_number,
42                 revision=build_number + 1000,
43                 is_green=build_number < 4
44             )
45             parsed_results = {LayoutTestResults.fail_key: failure(build_number)}
46             build._layout_test_results = LayoutTestResults(parsed_results)
47             return build
48         self.builder._fetch_build = _mock_fetch_build
49
50     def setUp(self):
51         self.buildbot = BuildBot()
52         self.builder = Builder(u"Test Builder \u2661", self.buildbot)
53         self._install_fetch_build(lambda build_number: ["test1", "test2"])
54
55     def test_find_regression_window(self):
56         regression_window = self.builder.find_regression_window(self.builder.build(10))
57         self.assertEqual(regression_window.build_before_failure().revision(), 1003)
58         self.assertEqual(regression_window.failing_build().revision(), 1004)
59
60         regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2)
61         self.assertEqual(regression_window.build_before_failure(), None)
62         self.assertEqual(regression_window.failing_build().revision(), 1008)
63
64     def test_none_build(self):
65         self.builder._fetch_build = lambda build_number: None
66         regression_window = self.builder.find_regression_window(self.builder.build(10))
67         self.assertEqual(regression_window.build_before_failure(), None)
68         self.assertEqual(regression_window.failing_build(), None)
69
70     def test_flaky_tests(self):
71         self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"])
72         regression_window = self.builder.find_regression_window(self.builder.build(10))
73         self.assertEqual(regression_window.build_before_failure().revision(), 1009)
74         self.assertEqual(regression_window.failing_build().revision(), 1010)
75
76     def test_failure_and_flaky(self):
77         self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
78         regression_window = self.builder.find_regression_window(self.builder.build(10))
79         self.assertEqual(regression_window.build_before_failure().revision(), 1003)
80         self.assertEqual(regression_window.failing_build().revision(), 1004)
81
82     def test_no_results(self):
83         self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"])
84         regression_window = self.builder.find_regression_window(self.builder.build(10))
85         self.assertEqual(regression_window.build_before_failure().revision(), 1003)
86         self.assertEqual(regression_window.failing_build().revision(), 1004)
87
88     def test_failure_after_flaky(self):
89         self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"])
90         regression_window = self.builder.find_regression_window(self.builder.build(10))
91         self.assertEqual(regression_window.build_before_failure().revision(), 1006)
92         self.assertEqual(regression_window.failing_build().revision(), 1007)
93
94     def test_find_blameworthy_regression_window(self):
95         self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004])
96         self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None)
97         # Flakey test avoidance requires at least 2 red builds:
98         self.assertEqual(self.builder.find_blameworthy_regression_window(4), None)
99         self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004])
100         # Green builder:
101         self.assertEqual(self.builder.find_blameworthy_regression_window(3), None)
102
103     def test_build_caching(self):
104         self.assertEqual(self.builder.build(10), self.builder.build(10))
105
106     def test_build_and_revision_for_filename(self):
107         expectations = {
108             "r47483 (1)/" : (47483, 1),
109             "r47483 (1).zip" : (47483, 1),
110         }
111         for filename, revision_and_build in expectations.items():
112             self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build)
113
114
115 class BuildTest(unittest.TestCase):
116     def test_layout_test_results(self):
117         build = Build(None, None, None, None)
118         build._fetch_results_html = lambda: None
119         # Test that layout_test_results() returns None if the fetch fails.
120         self.assertEqual(build.layout_test_results(), None)
121
122
123 class BuildBotTest(unittest.TestCase):
124
125     _example_one_box_status = '''
126     <table>
127     <tr>
128     <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td>
129       <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td>
130       <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td>
131     <tr>
132     <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td>
133       <td class="LastBuild box" >no build</td>
134       <td align="center" class="Activity building">building<br />< 1 min</td>
135     <tr>
136     <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td>
137       <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td>
138       <td align="center" class="Activity idle">idle<br />3 pending</td>
139     <tr>
140     <td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td>
141       <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td>
142       <td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td>
143     </table>
144 '''
145     _expected_example_one_box_parsings = [
146         {
147             'is_green': True,
148             'build_number' : 3693,
149             'name': u'Windows Debug (Tests)',
150             'built_revision': 47380,
151             'activity': 'building',
152             'pending_builds': 0,
153         },
154         {
155             'is_green': False,
156             'build_number' : None,
157             'name': u'SnowLeopard Intel Release',
158             'built_revision': None,
159             'activity': 'building',
160             'pending_builds': 0,
161         },
162         {
163             'is_green': False,
164             'build_number' : 654,
165             'name': u'Qt Linux Release',
166             'built_revision': 47383,
167             'activity': 'idle',
168             'pending_builds': 3,
169         },
170         {
171             'is_green': True,
172             'build_number' : 2090,
173             'name': u'Qt Windows 32-bit Debug',
174             'built_revision': 60563,
175             'activity': 'building',
176             'pending_builds': 0,
177         },
178     ]
179
180     def test_status_parsing(self):
181         buildbot = BuildBot()
182
183         soup = BeautifulSoup(self._example_one_box_status)
184         status_table = soup.find("table")
185         input_rows = status_table.findAll('tr')
186
187         for x in range(len(input_rows)):
188             status_row = input_rows[x]
189             expected_parsing = self._expected_example_one_box_parsings[x]
190
191             builder = buildbot._parse_builder_status_from_row(status_row)
192
193             # Make sure we aren't parsing more or less than we expect
194             self.assertEquals(builder.keys(), expected_parsing.keys())
195
196             for key, expected_value in expected_parsing.items():
197                 self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value)))
198
199     def test_core_builder_methods(self):
200         buildbot = BuildBot()
201
202         # Override builder_statuses function to not touch the network.
203         def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to.
204             return BuildBotTest._expected_example_one_box_parsings
205         buildbot.builder_statuses = example_builder_statuses
206
207         buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ]
208         self.assertEquals(buildbot.red_core_builders_names(), [])
209         self.assertTrue(buildbot.core_builders_are_green())
210
211         buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ]
212         self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ])
213         self.assertFalse(buildbot.core_builders_are_green())
214
215     def test_builder_name_regexps(self):
216         buildbot = BuildBot()
217
218         # For complete testing, this list should match the list of builders at build.webkit.org:
219         example_builders = [
220             {'name': u'Tiger Intel Release', },
221             {'name': u'Leopard Intel Release (Build)', },
222             {'name': u'Leopard Intel Release (Tests)', },
223             {'name': u'Leopard Intel Debug (Build)', },
224             {'name': u'Leopard Intel Debug (Tests)', },
225             {'name': u'SnowLeopard Intel Release (Build)', },
226             {'name': u'SnowLeopard Intel Release (Tests)', },
227             {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', },
228             {'name': u'SnowLeopard Intel Leaks', },
229             {'name': u'Windows Release (Build)', },
230             {'name': u'Windows Release (Tests)', },
231             {'name': u'Windows Debug (Build)', },
232             {'name': u'Windows Debug (Tests)', },
233             {'name': u'GTK Linux 32-bit Release', },
234             {'name': u'GTK Linux 32-bit Debug', },
235             {'name': u'GTK Linux 64-bit Debug', },
236             {'name': u'GTK Linux 64-bit Release', },
237             {'name': u'Qt Linux Release', },
238             {'name': u'Qt Linux Release minimal', },
239             {'name': u'Qt Linux ARMv5 Release', },
240             {'name': u'Qt Linux ARMv7 Release', },
241             {'name': u'Qt Windows 32-bit Release', },
242             {'name': u'Qt Windows 32-bit Debug', },
243             {'name': u'Chromium Linux Release', },
244             {'name': u'Chromium Mac Release', },
245             {'name': u'Chromium Win Release', },
246             {'name': u'Chromium Linux Release (Tests)', },
247             {'name': u'Chromium Mac Release (Tests)', },
248             {'name': u'Chromium Win Release (Tests)', },
249             {'name': u'New run-webkit-tests', },
250         ]
251         name_regexps = [
252             "SnowLeopard.*Build",
253             "SnowLeopard.*\(Test",
254             "Leopard",
255             "Tiger",
256             "Windows.*Build",
257             "GTK.*32",
258             "GTK.*64.*Debug",  # Disallow the 64-bit Release bot which is broken.
259             "Qt",
260             "Chromium.*Release$",
261         ]
262         expected_builders = [
263             {'name': u'Tiger Intel Release', },
264             {'name': u'Leopard Intel Release (Build)', },
265             {'name': u'Leopard Intel Release (Tests)', },
266             {'name': u'Leopard Intel Debug (Build)', },
267             {'name': u'Leopard Intel Debug (Tests)', },
268             {'name': u'SnowLeopard Intel Release (Build)', },
269             {'name': u'SnowLeopard Intel Release (Tests)', },
270             {'name': u'Windows Release (Build)', },
271             {'name': u'Windows Debug (Build)', },
272             {'name': u'GTK Linux 32-bit Release', },
273             {'name': u'GTK Linux 32-bit Debug', },
274             {'name': u'GTK Linux 64-bit Debug', },
275             {'name': u'Qt Linux Release', },
276             {'name': u'Qt Linux Release minimal', },
277             {'name': u'Qt Linux ARMv5 Release', },
278             {'name': u'Qt Linux ARMv7 Release', },
279             {'name': u'Qt Windows 32-bit Release', },
280             {'name': u'Qt Windows 32-bit Debug', },
281             {'name': u'Chromium Linux Release', },
282             {'name': u'Chromium Mac Release', },
283             {'name': u'Chromium Win Release', },
284         ]
285
286         # This test should probably be updated if the default regexp list changes
287         self.assertEquals(buildbot.core_builder_names_regexps, name_regexps)
288
289         builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps)
290         self.assertEquals(builders, expected_builders)
291
292     def test_builder_with_name(self):
293         buildbot = BuildBot()
294
295         builder = buildbot.builder_with_name("Test Builder")
296         self.assertEqual(builder.name(), "Test Builder")
297         self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder")
298         self.assertEqual(builder.url_encoded_name(), "Test%20Builder")
299         self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder")
300
301         # Override _fetch_build_dictionary function to not touch the network.
302         def mock_fetch_build_dictionary(self, build_number):
303             build_dictionary = {
304                 "sourceStamp": {
305                     "revision" : 2 * build_number,
306                     },
307                 "number" : int(build_number),
308                 "results" : build_number % 2, # 0 means pass
309             }
310             return build_dictionary
311         buildbot._fetch_build_dictionary = mock_fetch_build_dictionary
312
313         build = builder.build(10)
314         self.assertEqual(build.builder(), builder)
315         self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10")
316         self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29")
317         self.assertEqual(build.revision(), 20)
318         self.assertEqual(build.is_green(), True)
319
320         build = build.previous_build()
321         self.assertEqual(build.builder(), builder)
322         self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9")
323         self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29")
324         self.assertEqual(build.revision(), 18)
325         self.assertEqual(build.is_green(), False)
326
327         self.assertEqual(builder.build(None), None)
328
329     _example_directory_listing = '''
330 <h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1>
331
332 <table>
333         <tr class="alt">
334             <th>Filename</th>
335             <th>Size</th>
336             <th>Content type</th>
337             <th>Content encoding</th>
338         </tr>
339 <tr class="directory ">
340     <td><a href="r47483%20%281%29/"><b>r47483 (1)/</b></a></td>
341     <td><b></b></td>
342     <td><b>[Directory]</b></td>
343     <td><b></b></td>
344 </tr>
345 <tr class="file alt">
346     <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td>
347     <td>89K</td>
348     <td>[application/zip]</td>
349     <td></td>
350 </tr>
351 '''
352     _expected_files = [
353         {
354             "filename" : "r47483 (1)/",
355             "size" : "",
356             "type" : "[Directory]",
357             "encoding" : "",
358         },
359         {
360             "filename" : "r47484 (2).zip",
361             "size" : "89K",
362             "type" : "[application/zip]",
363             "encoding" : "",
364         },
365     ]
366
367     def test_parse_build_to_revision_map(self):
368         buildbot = BuildBot()
369         files = buildbot._parse_twisted_directory_listing(self._example_directory_listing)
370         self.assertEqual(self._expected_files, files)
371
372     # Revision, is_green
373     # Ordered from newest (highest number) to oldest.
374     fake_builder1 = [
375         [2, False],
376         [1, True],
377     ]
378     fake_builder2 = [
379         [2, False],
380         [1, True],
381     ]
382     fake_builders = [
383         fake_builder1,
384         fake_builder2,
385     ]
386     def _build_from_fake(self, fake_builder, index):
387         if index >= len(fake_builder):
388             return None
389         fake_build = fake_builder[index]
390         build = Build(
391             builder=fake_builder,
392             build_number=index,
393             revision=fake_build[0],
394             is_green=fake_build[1],
395         )
396         def mock_previous_build():
397             return self._build_from_fake(fake_builder, index + 1)
398         build.previous_build = mock_previous_build
399         return build
400
401     def _fake_builds_at_index(self, index):
402         return [self._build_from_fake(builder, index) for builder in self.fake_builders]
403
404     def test_last_green_revision(self):
405         buildbot = BuildBot()
406         def mock_builds_from_builders(only_core_builders):
407             return self._fake_builds_at_index(0)
408         buildbot._latest_builds_from_builders = mock_builds_from_builders
409         self.assertEqual(buildbot.last_green_revision(), 1)
410
411
412 if __name__ == '__main__':
413     unittest.main()