run-api-tests: Upload test results
authorjbedard@apple.com <jbedard@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 2 Apr 2019 02:57:05 +0000 (02:57 +0000)
committerjbedard@apple.com <jbedard@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 2 Apr 2019 02:57:05 +0000 (02:57 +0000)
https://bugs.webkit.org/show_bug.cgi?id=196323
<rdar://problem/49356714>

Reviewed by Lucas Forschler.

* Scripts/webkitpy/api_tests/manager.py:
(Manager):
(Manager.run): Upload results to a results database.
* Scripts/webkitpy/api_tests/run_api_tests.py:
(parse_args): Add upload arguments.
* Scripts/webkitpy/port/base.py:
(Port):
(Port.configuration_for_upload): Creates a configuration dictionary for uploading results.
(Port.commits_for_upload): Create a list of commits from the WebKit repository tests are run from along
with commits from any other associated repositories.
* Scripts/webkitpy/port/device.py:
(Device):
(Device.build_version): Access build_versoin of underlying platform device.
* Scripts/webkitpy/port/ios_simulator_unittest.py:
(IOSSimulatorTest):
(IOSSimulatorTest.test_configuration_for_upload):
* Scripts/webkitpy/port/device_port.py:
(DevicePort):
(DevicePort.configuration_for_upload): Devices are unique because their configuration is not
the same as the machine uploading results.
* Scripts/webkitpy/port/mac.py:
(MacPort):
(MacPort.configuration_for_upload): Define SDK in upload configuration for Mac.
* Scripts/webkitpy/port/mac_unittest.py:
(MacTest):
(MacTest.test_configuration_for_upload):
* Scripts/webkitpy/xcode/simulated_device.py:
(SimulatedDeviceManager._create_device_with_runtime):
(SimulatedDevice.__init__): Create simulated device with a build_version.
* Scripts/webkitpy/xcode/simulated_device_unittest.py:
(test_existing_simulator):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@243732 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/api_tests/manager.py
Tools/Scripts/webkitpy/api_tests/run_api_tests.py
Tools/Scripts/webkitpy/port/base.py
Tools/Scripts/webkitpy/port/device.py
Tools/Scripts/webkitpy/port/device_port.py
Tools/Scripts/webkitpy/port/ios_simulator_unittest.py
Tools/Scripts/webkitpy/port/mac.py
Tools/Scripts/webkitpy/port/mac_unittest.py
Tools/Scripts/webkitpy/xcode/simulated_device.py
Tools/Scripts/webkitpy/xcode/simulated_device_unittest.py

index 8b0cbfb..4131961 100644 (file)
@@ -1,3 +1,43 @@
+2019-04-01  Jonathan Bedard  <jbedard@apple.com>
+
+        run-api-tests: Upload test results
+        https://bugs.webkit.org/show_bug.cgi?id=196323
+        <rdar://problem/49356714>
+
+        Reviewed by Lucas Forschler.
+
+        * Scripts/webkitpy/api_tests/manager.py:
+        (Manager):
+        (Manager.run): Upload results to a results database.
+        * Scripts/webkitpy/api_tests/run_api_tests.py:
+        (parse_args): Add upload arguments.
+        * Scripts/webkitpy/port/base.py:
+        (Port):
+        (Port.configuration_for_upload): Creates a configuration dictionary for uploading results.
+        (Port.commits_for_upload): Create a list of commits from the WebKit repository tests are run from along
+        with commits from any other associated repositories.
+        * Scripts/webkitpy/port/device.py:
+        (Device):
+        (Device.build_version): Access build_versoin of underlying platform device.
+        * Scripts/webkitpy/port/ios_simulator_unittest.py:
+        (IOSSimulatorTest):
+        (IOSSimulatorTest.test_configuration_for_upload):
+        * Scripts/webkitpy/port/device_port.py:
+        (DevicePort):
+        (DevicePort.configuration_for_upload): Devices are unique because their configuration is not
+        the same as the machine uploading results.
+        * Scripts/webkitpy/port/mac.py:
+        (MacPort):
+        (MacPort.configuration_for_upload): Define SDK in upload configuration for Mac.
+        * Scripts/webkitpy/port/mac_unittest.py:
+        (MacTest):
+        (MacTest.test_configuration_for_upload):
+        * Scripts/webkitpy/xcode/simulated_device.py:
+        (SimulatedDeviceManager._create_device_with_runtime):
+        (SimulatedDevice.__init__): Create simulated device with a build_version.
+        * Scripts/webkitpy/xcode/simulated_device_unittest.py:
+        (test_existing_simulator):
+
 2019-04-01  Aakash Jain  <aakash_jain@apple.com>
 
         Remove extra newline characters (Follow-up fix to r243707)
index 9f371c2..3237e92 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Apple Inc. All rights reserved.
+# Copyright (C) 2018-2019 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 
 import json
 import logging
-import os
+import time
 
 from webkitpy.api_tests.runner import Runner
 from webkitpy.common.system.executive import ScriptError
+from webkitpy.results.upload import Upload
+
 from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
 
 _log = logging.getLogger(__name__)
@@ -39,6 +41,7 @@ class Manager(object):
     FAILED_BUILD_CHECK = 1
     FAILED_COLLECT_TESTS = 2
     FAILED_TESTS = 3
+    FAILED_UPLOAD = 4
 
     def __init__(self, port, options, stream):
         self._port = port
@@ -154,6 +157,8 @@ class Manager(object):
             if not self.host.filesystem.isdir(self.host.filesystem.dirname(json_output)) or self.host.filesystem.isdir(json_output):
                 raise RuntimeError('Cannot write to {}'.format(json_output))
 
+        start_time = time.time()
+
         self._stream.write_update('Checking build ...')
         if not self._port.check_api_test_build(self._binaries_for_arguments(args)):
             _log.error('Build check failed')
@@ -185,6 +190,8 @@ class Manager(object):
             # If we receive a KeyboardInterrupt, print results.
             self._stream.writeln('')
 
+        end_time = time.time()
+
         successful = runner.result_map_by_status(runner.STATUS_PASSED)
         disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED))
         _log.info('Ran {} tests of {} with {} successful'.format(len(runner.results) - disabled, len(test_names), len(successful)))
@@ -197,42 +204,73 @@ class Manager(object):
         }
 
         self._stream.writeln('-' * 30)
+        result = Manager.SUCCESS
         if len(successful) + disabled == len(test_names):
             self._stream.writeln('All tests successfully passed!')
             if json_output:
                 self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
-            return Manager.SUCCESS
+        else:
+            self._stream.writeln('Test suite failed')
+            self._stream.writeln('')
 
-        self._stream.writeln('Test suite failed')
-        self._stream.writeln('')
+            skipped = []
+            for test in test_names:
+                if test not in runner.results:
+                    skipped.append(test)
+                    result_dictionary['Skipped'].append({'name': test, 'output': None})
+            if skipped:
+                self._stream.writeln('Skipped {} tests'.format(len(skipped)))
+                self._stream.writeln('')
+                if self._options.verbose:
+                    for test in skipped:
+                        self._stream.writeln('    {}'.format(test))
+
+            self._print_tests_result_with_status(runner.STATUS_FAILED, runner)
+            self._print_tests_result_with_status(runner.STATUS_CRASHED, runner)
+            self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner)
+
+            for test, result in runner.results.iteritems():
+                status_to_string = {
+                    runner.STATUS_FAILED: 'Failed',
+                    runner.STATUS_CRASHED: 'Crashed',
+                    runner.STATUS_TIMEOUT: 'Timedout',
+                }.get(result[0])
+                if not status_to_string:
+                    continue
+                result_dictionary[status_to_string].append({'name': test, 'output': result[1]})
 
-        skipped = []
-        for test in test_names:
-            if test not in runner.results:
-                skipped.append(test)
-                result_dictionary['Skipped'].append({'name': test, 'output': None})
-        if skipped:
-            self._stream.writeln('Skipped {} tests'.format(len(skipped)))
-            self._stream.writeln('')
-            if self._options.verbose:
-                for test in skipped:
-                    self._stream.writeln('    {}'.format(test))
-
-        self._print_tests_result_with_status(runner.STATUS_FAILED, runner)
-        self._print_tests_result_with_status(runner.STATUS_CRASHED, runner)
-        self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner)
-
-        for test, result in runner.results.iteritems():
-            status_to_string = {
-                runner.STATUS_FAILED: 'Failed',
-                runner.STATUS_CRASHED: 'Crashed',
-                runner.STATUS_TIMEOUT: 'Timedout',
-            }.get(result[0])
-            if not status_to_string:
-                continue
-            result_dictionary[status_to_string].append({'name': test, 'output': result[1]})
+            if json_output:
+                self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
 
-        if json_output:
-            self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
+            result = Manager.FAILED_TESTS
+
+        if self._options.report_urls:
+            self._stream.writeln('\n')
+            self._stream.write_update('Preparing upload data ...')
+
+            status_to_test_result = {
+                runner.STATUS_PASSED: None,
+                runner.STATUS_FAILED: Upload.Expectations.FAIL,
+                runner.STATUS_CRASHED: Upload.Expectations.CRASH,
+                runner.STATUS_TIMEOUT: Upload.Expectations.TIMEOUT,
+            }
+            upload = Upload(
+                suite='api-tests',
+                configuration=self._port.configuration_for_upload(self._port.target_host(0)),
+                details=Upload.create_details(options=self._options),
+                commits=self._port.commits_for_upload(),
+                run_stats=Upload.create_run_stats(
+                    start_time=start_time,
+                    end_time=end_time,
+                    tests_skipped=len(result_dictionary['Skipped']),
+                ),
+                results={test: Upload.create_test_result(actual=status_to_test_result[result[0]])
+                         for test, result in runner.results.iteritems() if result[0] in status_to_test_result},
+            )
+            for url in self._options.report_urls:
+                self._stream.write_update('Uploading to {} ...'.format(url))
+                if not upload.upload(url, log_line_func=self._stream.writeln):
+                    result = Manager.FAILED_UPLOAD
+            self._stream.writeln('Uploads completed!')
 
-        return Manager.FAILED_TESTS
+        return result
index 76a021e..c0fc7b0 100644 (file)
@@ -30,6 +30,7 @@ from webkitpy.api_tests.manager import Manager
 from webkitpy.common.host import Host
 from webkitpy.layout_tests.views.metered_stream import MeteredStream
 from webkitpy.port import configuration_options, platform_options, base, win
+from webkitpy.results.options import upload_options
 
 EXCEPTIONAL_EXIT_STATUS = -1
 INTERRUPT_EXIT_STATUS = -2
@@ -134,6 +135,7 @@ def parse_args(args):
         optparse.make_option('--force', action='store_true', default=False,
                              help='Run all tests, even DISABLED tests'),
     ]))
+    option_group_definitions.append(('Upload Options', upload_options()))
 
     option_parser = optparse.OptionParser(
         usage='run-api-tests [options] [<test names>...]',
index eb39988..f5c9476 100644 (file)
@@ -45,6 +45,7 @@ from functools import partial
 
 from webkitpy.common import find_files
 from webkitpy.common import read_checksum_from_png
+from webkitpy.common.checkout.scm.detection import SCMDetector
 from webkitpy.common.memoized import memoized
 from webkitpy.common.prettypatch import PrettyPatch
 from webkitpy.common.system import path, pemfile
@@ -61,6 +62,7 @@ from webkitpy.port.factory import PortFactory
 from webkitpy.layout_tests.servers import apache_http_server, http_server, http_server_base
 from webkitpy.layout_tests.servers import web_platform_test_server
 from webkitpy.layout_tests.servers import websocket_server
+from webkitpy.results.upload import Upload
 
 _log = logging.getLogger(__name__)
 
@@ -1634,3 +1636,38 @@ class Port(object):
         # This is overridden by ports that need to do work in the parent process after a worker subprocess is spawned,
         # such as closing file descriptors that were implicitly cloned to the worker.
         pass
+
+    def configuration_for_upload(self, host=None):
+        configuration = self.test_configuration()
+        host = self.host or host
+
+        return Upload.create_configuration(
+            platform=host.platform.os_name,
+            version=str(host.platform.os_version),
+            version_name=host.platform.os_version_name(INTERNAL_TABLE) or host.platform.os_version_name(),
+            architecture=configuration.architecture,
+            style='guard-malloc' if self.get_option('guard_malloc') else configuration.build_type,
+            sdk=host.platform.build_version(),
+        )
+
+    @memoized
+    def commits_for_upload(self):
+        self.host.initialize_scm()
+
+        if port_config.apple_additions() and getattr(port_config.apple_additions(), 'repos', False):
+            repos = port_config.apple_additions().repos()
+        else:
+            repos = {}
+
+        up = os.path.dirname
+        repos['webkit'] = up(up(up(up(up(os.path.abspath(__file__))))))
+
+        commits = []
+        for repo_id, path in repos.iteritems():
+            scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(path)
+            commits.append(Upload.create_commit(
+                repository_id=repo_id,
+                id=scm.native_revision(path),
+                branch=scm.native_branch(path),
+            ))
+        return commits
index f49fdef..bf8e8e5 100644 (file)
@@ -101,6 +101,10 @@ class Device(object):
     def device_type(self):
         return self.platform_device.device_type
 
+    @property
+    def build_version(self):
+        return self.platform_device.build_version
+
     def __nonzero__(self):
         return self.platform_device is not None
 
index 8fbca5c..3d4c236 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2018 Apple Inc. All rights reserved.
+# Copyright (C) 2018-2019 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 import logging
 import traceback
 
+from webkitpy.common.version_name_map import VersionNameMap, PUBLIC_TABLE, INTERNAL_TABLE
 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
 from webkitpy.port.darwin import DarwinPort
 from webkitpy.port.simulator_process import SimulatorProcess
+from webkitpy.results.upload import Upload
 from webkitpy.xcode.device_type import DeviceType
 from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
 
@@ -238,3 +240,29 @@ class DevicePort(DarwinPort):
 
     def device_version(self):
         raise NotImplementedError
+
+    def configuration_for_upload(self, host=None):
+        configuration = self.test_configuration()
+
+        device_type = host.device_type if host else self.DEVICE_TYPE
+        model = device_type.hardware_family
+        if model and device_type.hardware_type:
+            model += ' {}'.format(device_type.hardware_type)
+
+        version = self.device_version()
+        version_name = None
+        for table in [INTERNAL_TABLE, PUBLIC_TABLE]:
+            version_name = VersionNameMap.map(self.host.platform).to_name(version, platform=device_type.software_variant.lower(), table=table)
+            if version_name:
+                break
+
+        return Upload.create_configuration(
+            platform=device_type.software_variant.lower(),
+            is_simulator=self.DEVICE_MANAGER == SimulatedDeviceManager,
+            version=str(version),
+            version_name=version_name,
+            architecture=configuration.architecture,
+            style='guard-malloc' if self.get_option('guard_malloc') else configuration.build_type,
+            model=model,
+            sdk=host.build_version if host else None,
+        )
index 7d4d54d..3d8138a 100644 (file)
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016 Apple Inc. All rights reserved.
+# Copyright (C) 2014-2019 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -161,3 +161,17 @@ class IOSSimulatorTest(ios_testcase.IOSTest):
     def test_max_child_processes(self):
         port = self.make_port()
         self.assertEqual(port.max_child_processes(DeviceType.from_string('Apple Watch')), 0)
+
+    def test_configuration_for_upload(self):
+        port = self.make_port()
+        self.assertEqual(
+            dict(
+                platform='ios',
+                is_simulator=True,
+                architecture='x86_64',
+                version='11',
+                version_name='iOS 11',
+                style='release',
+            ),
+            port.configuration_for_upload(),
+        )
index bcf8a12..50f4c58 100644 (file)
@@ -1,5 +1,5 @@
 # Copyright (C) 2011 Google Inc. All rights reserved.
-# Copyright (C) 2012, 2013, 2016 Apple Inc. All rights reserved.
+# Copyright (C) 2012-2019 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -279,3 +279,14 @@ class MacPort(DarwinPort):
         worthless_patterns.append((re.compile('.*<<<< VMC >>>>.*\n'), ''))
         worthless_patterns.append((re.compile('.*<<< FFR_Common >>>.*\n'), ''))
         return worthless_patterns
+
+    def configuration_for_upload(self, host=None):
+        host = host or self.host
+        configuration = super(MacPort, self).configuration_for_upload(host=host)
+
+        output = host.executive.run_command(['/usr/sbin/sysctl', 'hw.model']).rstrip()
+        match = re.match(r'hw.model: (?P<model>.*)', output)
+        if match:
+            configuration['model'] = match.group('model')
+
+        return configuration
index 8908df8..5a3edf4 100644 (file)
@@ -1,5 +1,5 @@
 # Copyright (C) 2010 Google Inc. All rights reserved.
-# Copyright (C) 2014-2016 Apple Inc. All rights reserved.
+# Copyright (C) 2014-2019 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
@@ -224,3 +224,18 @@ class MacTest(darwin_testcase.DarwinTest):
 
         port = self.make_port(options=MockOptions(webkit_test_runner=False), port_name='mac-wk2')
         self.assertEqual(port.driver_name(), 'WebKitTestRunner')
+
+    def test_configuration_for_upload(self):
+        port = self.make_port()
+        self.assertEqual(
+            dict(
+                platform='mac',
+                is_simulator=False,
+                architecture='x86_64',
+                version='10.7',
+                version_name='Lion',
+                sdk='17A405',
+                style='release',
+            ),
+            port.configuration_for_upload(),
+        )
index 0dfee77..16b0233 100644 (file)
@@ -113,6 +113,7 @@ class SimulatedDeviceManager(object):
             udid=device_info['udid'],
             host=host,
             device_type=device_type,
+            build_version=runtime.build_version,
         ))
         SimulatedDeviceManager.AVAILABLE_DEVICES.append(result)
         return result
@@ -488,12 +489,13 @@ class SimulatedDevice(object):
         'SHUTTING DOWN',
     ]
 
-    def __init__(self, name, udid, host, device_type):
+    def __init__(self, name, udid, host, device_type, build_version):
         assert device_type.software_version
 
         self.name = name
         self.udid = udid
         self.device_type = device_type
+        self.build_version = build_version
         self._state = SimulatedDevice.DeviceState.SHUTTING_DOWN
         self._last_updated_state = time.time()
 
index 943638d..e3b18d7 100644 (file)
@@ -605,6 +605,7 @@ class SimulatedDeviceTest(unittest.TestCase):
 
         self.assertEquals(1, len(SimulatedDeviceManager.INITIALIZED_DEVICES))
         self.assertEquals('34FB476C-6FA0-43C8-8945-1BD7A4EBF0DE', SimulatedDeviceManager.INITIALIZED_DEVICES[0].udid)
+        self.assertEquals('15A8401', SimulatedDeviceManager.INITIALIZED_DEVICES[0].build_version)
         self.assertEquals(SimulatedDevice.DeviceState.BOOTED, SimulatedDeviceManager.INITIALIZED_DEVICES[0].platform_device.state())
 
         SimulatedDeviceManager.tear_down(host)