Tools/Scripts/webkitpy/tool/steps/steps_unittest.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 # Copyright (C) 2017 Apple Inc. All rights reserved.
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 import unittest
  31
  32 from webkitpy.common.system.executive import ScriptError
  33 from webkitpy.common.system.outputcapture import OutputCapture
  34 from webkitpy.common.config.ports import DeprecatedPort
  35 from webkitpy.tool.mocktool import MockOptions, MockTool
  36
  37 from webkitpy.tool import steps
  38
  39
  40 class StepsTest(unittest.TestCase):
  41
  42     def setUp(self):
  43         # Port._build_path() calls Tools/Scripts/webkit-build-directory and caches the result. When capturing output,
  44         # this can cause the first invocation of Port._build_path() to have more output than subsequent invocations.
  45         # This may cause test flakiness when test order changes. By explicitly calling Port._build_path() before running
  46         # tests in this suite, we avoid such flakiness.
  47         MockTool().port_factory.get(options=self._step_options())._build_path()
  48
  49     def _step_options(self):
  50         options = MockOptions()
  51         options.group = None
  52         options.non_interactive = True
  53         options.port = 'MOCK port'
  54         options.quiet = True
  55         options.test = True
  56         options.iterate_on_new_tests = 0
  57         options.root = '/tmp'
  58         return options
  59
  60     def _run_step(self, step, tool=None, options=None, state=None):
  61         if not tool:
  62             tool = MockTool()
  63         if not options:
  64             options = self._step_options()
  65         if not state:
  66             state = {}
  67         step(tool, options).run(state)
  68
  69     def test_update_step(self):
  70         tool = MockTool()
  71         options = self._step_options()
  72         options.update = True
  73         expected_logs = "Updating working directory\n"
  74         OutputCapture().assert_outputs(self, self._run_step, [steps.Update, tool, options], expected_logs=expected_logs)
  75
  76     def test_prompt_for_bug_or_title_step(self):
  77         tool = MockTool()
  78         tool.user.prompt = lambda message: 50000
  79         self._run_step(steps.PromptForBugOrTitle, tool=tool)
  80
  81     def _post_diff_options(self):
  82         options = self._step_options()
  83         options.git_commit = None
  84         options.description = None
  85         options.comment = None
  86         options.review = True
  87         options.request_commit = False
  88         options.open_bug = True
  89         return options
  90
  91     def _assert_step_output_with_bug(self, step, bug_id, expected_logs, options=None):
  92         state = {'bug_id': bug_id}
  93         OutputCapture().assert_outputs(self, self._run_step, [step, MockTool(), options, state], expected_logs=expected_logs)
  94
  95     def _assert_post_diff_output_for_bug(self, step, bug_id, expected_logs):
  96         self._assert_step_output_with_bug(step, bug_id, expected_logs, self._post_diff_options())
  97
  98     def test_post_diff(self):
  99         expected_logs = "MOCK add_patch_to_bug: bug_id=78, description=Patch, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False\nMOCK: user.open_url: http://example.com/78\n"
 100         self._assert_post_diff_output_for_bug(steps.PostDiff, 78, expected_logs)
 101
 102     def test_post_diff_for_commit(self):
 103         expected_logs = "MOCK add_patch_to_bug: bug_id=78, description=Patch for landing, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=True\n"
 104         self._assert_post_diff_output_for_bug(steps.PostDiffForCommit, 78, expected_logs)
 105
 106     def test_ensure_bug_is_open_and_assigned(self):
 107         expected_logs = "MOCK reopen_bug 50004 with comment 'Reopening to attach new patch.'\n"
 108         self._assert_step_output_with_bug(steps.EnsureBugIsOpenAndAssigned, 50004, expected_logs)
 109         expected_logs = "MOCK reassign_bug: bug_id=50002, assignee=None\n"
 110         self._assert_step_output_with_bug(steps.EnsureBugIsOpenAndAssigned, 50002, expected_logs)
 111
 112     def test_runtests_args(self):
 113         mock_options = self._step_options()
 114         mock_options.non_interactive = False
 115         mock_options.build_style = "release"
 116         step = steps.RunTests(MockTool(log_executive=True), mock_options)
 117         tool = MockTool(log_executive=True)
 118         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 119         tool._deprecated_port = DeprecatedPort()
 120         step = steps.RunTests(tool, mock_options)
 121         expected_logs = """Running Python unit tests
 122 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy'], cwd=/mock-checkout
 123 Running Perl unit tests
 124 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitperl'], cwd=/mock-checkout
 125 Running JavaScriptCore tests
 126 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast'], cwd=/mock-checkout
 127 Running run-webkit-tests
 128 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-webkit-tests', '--release', '--quiet'], cwd=/mock-checkout
 129 """
 130         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 131
 132     def test_runtests_debug_args(self):
 133         mock_options = self._step_options()
 134         mock_options.non_interactive = False
 135         mock_options.build_style = "debug"
 136         step = steps.RunTests(MockTool(log_executive=True), mock_options)
 137         tool = MockTool(log_executive=True)
 138         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 139         tool._deprecated_port = DeprecatedPort()
 140         step = steps.RunTests(tool, mock_options)
 141         expected_logs = """Running Python unit tests
 142 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy'], cwd=/mock-checkout
 143 Running Perl unit tests
 144 MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitperl'], cwd=/mock-checkout
 145 Running JavaScriptCore tests
 146 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast'], cwd=/mock-checkout
 147 Running run-webkit-tests
 148 MOCK run_and_throw_if_fail: ['Tools/Scripts/run-webkit-tests', '--debug', '--quiet'], cwd=/mock-checkout
 149 """
 150         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 151
 152     def test_runtests_jsc(self):
 153         mock_options = self._step_options()
 154         mock_options.non_interactive = False
 155         mock_options.build_style = "release"
 156         mock_options.group = "jsc"
 157         step = steps.RunTests(MockTool(log_executive=True), mock_options)
 158         tool = MockTool(log_executive=True)
 159         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 160         tool._deprecated_port = DeprecatedPort()
 161         step = steps.RunTests(tool, mock_options)
 162         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--release', '--json-output=/tmp/jsc_test_results.json'], cwd=/mock-checkout
 163 """
 164         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 165
 166     def test_runtests_jsc_debug(self):
 167         mock_options = self._step_options()
 168         mock_options.non_interactive = False
 169         mock_options.build_style = "debug"
 170         mock_options.group = "jsc"
 171         tool = MockTool(log_executive=True)
 172         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 173         tool._deprecated_port = DeprecatedPort()
 174         step = steps.RunTests(tool, mock_options)
 175         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests', '--no-fail-fast', '--debug', '--json-output=/tmp/jsc_test_results.json'], cwd=/mock-checkout
 176 """
 177         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 178
 179     def test_build_jsc_debug(self):
 180         mock_options = self._step_options()
 181         mock_options.non_interactive = False
 182         mock_options.build_style = "debug"
 183         mock_options.build = True
 184         mock_options.architecture = True
 185         mock_options.group = "jsc"
 186         tool = MockTool(log_executive=True)
 187         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 188         tool._deprecated_port = DeprecatedPort()
 189         step = steps.Build(tool, mock_options)
 190         expected_logs = """Building WebKit
 191 MOCK run_and_throw_if_fail: ['Tools/Scripts/build-jsc', '--debug', 'ARCHS=True'], cwd=/mock-checkout, env={'TERM': 'dumb', 'MOCK_ENVIRON_COPY': '1'}
 192 """
 193         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 194
 195     def test_build_jsc(self):
 196         mock_options = self._step_options()
 197         mock_options.non_interactive = False
 198         mock_options.build_style = "release"
 199         mock_options.build = True
 200         mock_options.architecture = True
 201         mock_options.group = "jsc"
 202         tool = MockTool(log_executive=True)
 203         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 204         tool._deprecated_port = DeprecatedPort()
 205         step = steps.Build(tool, mock_options)
 206         expected_logs = """Building WebKit
 207 MOCK run_and_throw_if_fail: ['Tools/Scripts/build-jsc', '--release', 'ARCHS=True'], cwd=/mock-checkout, env={'TERM': 'dumb', 'MOCK_ENVIRON_COPY': '1'}
 208 """
 209         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 210
 211     def test_patch_relevant(self):
 212         self.maxDiff = None
 213         mock_options = self._step_options()
 214         tool = MockTool(log_executive=True)
 215         tool.scm()._mockChangedFiles = ["JSTests/MockFile1", "ChangeLog"]
 216         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 217         tool._deprecated_port = DeprecatedPort()
 218         step = steps.CheckPatchRelevance(tool, mock_options)
 219         expected_logs = """Checking relevance of patch
 220 """
 221         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 222
 223     def test_patch_relevant_jsc(self):
 224         self.maxDiff = None
 225         mock_options = self._step_options()
 226         mock_options.group = "jsc"
 227         tool = MockTool(log_executive=True)
 228         tool.scm()._mockChangedFiles = ["JSTests/MockFile1", "ChangeLog"]
 229         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 230         tool._deprecated_port = DeprecatedPort()
 231         step = steps.CheckPatchRelevance(tool, mock_options)
 232         expected_logs = """Checking relevance of patch
 233 """
 234         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 235
 236     def test_patch_not_relevant_jsc(self):
 237         self.maxDiff = None
 238         mock_options = self._step_options()
 239         mock_options.group = "jsc"
 240         tool = MockTool(log_executive=True)
 241         tool.scm()._mockChangedFiles = ["Tools/ChangeLog", "Tools/Scripts/webkitpy/tool/steps/steps_unittest.py"]
 242         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 243         tool._deprecated_port = DeprecatedPort()
 244         step = steps.CheckPatchRelevance(tool, mock_options)
 245         expected_logs = """Checking relevance of patch
 246 This patch does not have relevant changes.
 247 """
 248         with self.assertRaises(ScriptError):
 249             OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 250
 251     def test_runtests_bindings(self):
 252         mock_options = self._step_options()
 253         mock_options.non_interactive = False
 254         mock_options.group = "bindings"
 255         step = steps.RunTests(MockTool(log_executive=True), mock_options)
 256         tool = MockTool(log_executive=True)
 257         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 258         tool._deprecated_port = DeprecatedPort()
 259         step = steps.RunTests(tool, mock_options)
 260         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-bindings-tests', '--json-output=/tmp/bindings_test_results.json'], cwd=/mock-checkout
 261 """
 262         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 263
 264     def test_runtests_webkitpy(self):
 265         mock_options = self._step_options()
 266         mock_options.non_interactive = False
 267         mock_options.group = "webkitpy"
 268         step = steps.RunTests(MockTool(log_executive=True), mock_options)
 269         tool = MockTool(log_executive=True)
 270         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 271         tool._deprecated_port = DeprecatedPort()
 272         step = steps.RunTests(tool, mock_options)
 273         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitpy', '--json-output=/tmp/python-unittest-results/webkitpy_test_results.json'], cwd=/mock-checkout
 274 """
 275         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 276
 277     def test_patch_relevant_bindings(self):
 278         self.maxDiff = None
 279         mock_options = self._step_options()
 280         mock_options.group = "bindings"
 281         tool = MockTool(log_executive=True)
 282         tool.scm()._mockChangedFiles = ["Source/WebCore/features.json", "Source/ChangeLog"]
 283         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 284         tool._deprecated_port = DeprecatedPort()
 285         step = steps.CheckPatchRelevance(tool, mock_options)
 286         expected_logs = """Checking relevance of patch
 287 """
 288         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 289
 290     def test_patch_not_relevant_bindings(self):
 291         self.maxDiff = None
 292         mock_options = self._step_options()
 293         mock_options.group = "bindings"
 294         tool = MockTool(log_executive=True)
 295         tool.scm()._mockChangedFiles = ["Source/JavaScriptCore/Makefile", "Source/ChangeLog"]
 296         # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment.
 297         tool._deprecated_port = DeprecatedPort()
 298         step = steps.CheckPatchRelevance(tool, mock_options)
 299         expected_logs = """Checking relevance of patch
 300 This patch does not have relevant changes.
 301 """
 302
 303     def test_runtests_api(self):
 304         mock_options = self._step_options()
 305         mock_options.non_interactive = False
 306         mock_options.build_style = "release"
 307         mock_options.group = "api"
 308         step = steps.RunTests(MockTool(log_executive=True), mock_options)
 309         tool = MockTool(log_executive=True)
 310         tool._deprecated_port = DeprecatedPort()
 311         step = steps.RunTests(tool, mock_options)
 312         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-api-tests', '--release', '--json-output=/tmp/api_test_results.json'], cwd=/mock-checkout
 313 """
 314         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)
 315
 316     def test_runtests_api_debug(self):
 317         mock_options = self._step_options()
 318         mock_options.non_interactive = False
 319         mock_options.build_style = "debug"
 320         mock_options.group = "api"
 321         step = steps.RunTests(MockTool(log_executive=True), mock_options)
 322         tool = MockTool(log_executive=True)
 323         tool._deprecated_port = DeprecatedPort()
 324         step = steps.RunTests(tool, mock_options)
 325         expected_logs = """MOCK run_and_throw_if_fail: ['Tools/Scripts/run-api-tests', '--debug', '--json-output=/tmp/api_test_results.json'], cwd=/mock-checkout
 326 """
 327         OutputCapture().assert_outputs(self, step.run, [{}], expected_logs=expected_logs)