https://bugs.webkit.org/show_bug.cgi?id=196323
<rdar://problem/
49356714>
Reviewed by Lucas Forschler.
* Scripts/webkitpy/api_tests/manager.py:
(Manager):
(Manager.run): Upload results to a results database.
* Scripts/webkitpy/api_tests/run_api_tests.py:
(parse_args): Add upload arguments.
* Scripts/webkitpy/port/base.py:
(Port):
(Port.configuration_for_upload): Creates a configuration dictionary for uploading results.
(Port.commits_for_upload): Create a list of commits from the WebKit repository tests are run from along
with commits from any other associated repositories.
* Scripts/webkitpy/port/device.py:
(Device):
(Device.build_version): Access build_versoin of underlying platform device.
* Scripts/webkitpy/port/ios_simulator_unittest.py:
(IOSSimulatorTest):
(IOSSimulatorTest.test_configuration_for_upload):
* Scripts/webkitpy/port/device_port.py:
(DevicePort):
(DevicePort.configuration_for_upload): Devices are unique because their configuration is not
the same as the machine uploading results.
* Scripts/webkitpy/port/mac.py:
(MacPort):
(MacPort.configuration_for_upload): Define SDK in upload configuration for Mac.
* Scripts/webkitpy/port/mac_unittest.py:
(MacTest):
(MacTest.test_configuration_for_upload):
* Scripts/webkitpy/xcode/simulated_device.py:
(SimulatedDeviceManager._create_device_with_runtime):
(SimulatedDevice.__init__): Create simulated device with a build_version.
* Scripts/webkitpy/xcode/simulated_device_unittest.py:
(test_existing_simulator):
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@243732
268f45cc-cd09-0410-ab3c-
d52691b4dbfc
+2019-04-01 Jonathan Bedard <jbedard@apple.com>
+
+ run-api-tests: Upload test results
+ https://bugs.webkit.org/show_bug.cgi?id=196323
+ <rdar://problem/49356714>
+
+ Reviewed by Lucas Forschler.
+
+ * Scripts/webkitpy/api_tests/manager.py:
+ (Manager):
+ (Manager.run): Upload results to a results database.
+ * Scripts/webkitpy/api_tests/run_api_tests.py:
+ (parse_args): Add upload arguments.
+ * Scripts/webkitpy/port/base.py:
+ (Port):
+ (Port.configuration_for_upload): Creates a configuration dictionary for uploading results.
+ (Port.commits_for_upload): Create a list of commits from the WebKit repository tests are run from along
+ with commits from any other associated repositories.
+ * Scripts/webkitpy/port/device.py:
+ (Device):
+ (Device.build_version): Access build_versoin of underlying platform device.
+ * Scripts/webkitpy/port/ios_simulator_unittest.py:
+ (IOSSimulatorTest):
+ (IOSSimulatorTest.test_configuration_for_upload):
+ * Scripts/webkitpy/port/device_port.py:
+ (DevicePort):
+ (DevicePort.configuration_for_upload): Devices are unique because their configuration is not
+ the same as the machine uploading results.
+ * Scripts/webkitpy/port/mac.py:
+ (MacPort):
+ (MacPort.configuration_for_upload): Define SDK in upload configuration for Mac.
+ * Scripts/webkitpy/port/mac_unittest.py:
+ (MacTest):
+ (MacTest.test_configuration_for_upload):
+ * Scripts/webkitpy/xcode/simulated_device.py:
+ (SimulatedDeviceManager._create_device_with_runtime):
+ (SimulatedDevice.__init__): Create simulated device with a build_version.
+ * Scripts/webkitpy/xcode/simulated_device_unittest.py:
+ (test_existing_simulator):
+
2019-04-01 Aakash Jain <aakash_jain@apple.com>
Remove extra newline characters (Follow-up fix to r243707)
-# Copyright (C) 2018 Apple Inc. All rights reserved.
+# Copyright (C) 2018-2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
import json
import logging
-import os
+import time
from webkitpy.api_tests.runner import Runner
from webkitpy.common.system.executive import ScriptError
+from webkitpy.results.upload import Upload
+
from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
_log = logging.getLogger(__name__)
FAILED_BUILD_CHECK = 1
FAILED_COLLECT_TESTS = 2
FAILED_TESTS = 3
+ FAILED_UPLOAD = 4
def __init__(self, port, options, stream):
self._port = port
if not self.host.filesystem.isdir(self.host.filesystem.dirname(json_output)) or self.host.filesystem.isdir(json_output):
raise RuntimeError('Cannot write to {}'.format(json_output))
+ start_time = time.time()
+
self._stream.write_update('Checking build ...')
if not self._port.check_api_test_build(self._binaries_for_arguments(args)):
_log.error('Build check failed')
# If we receive a KeyboardInterrupt, print results.
self._stream.writeln('')
+ end_time = time.time()
+
successful = runner.result_map_by_status(runner.STATUS_PASSED)
disabled = len(runner.result_map_by_status(runner.STATUS_DISABLED))
_log.info('Ran {} tests of {} with {} successful'.format(len(runner.results) - disabled, len(test_names), len(successful)))
}
self._stream.writeln('-' * 30)
+ result = Manager.SUCCESS
if len(successful) + disabled == len(test_names):
self._stream.writeln('All tests successfully passed!')
if json_output:
self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
- return Manager.SUCCESS
+ else:
+ self._stream.writeln('Test suite failed')
+ self._stream.writeln('')
- self._stream.writeln('Test suite failed')
- self._stream.writeln('')
+ skipped = []
+ for test in test_names:
+ if test not in runner.results:
+ skipped.append(test)
+ result_dictionary['Skipped'].append({'name': test, 'output': None})
+ if skipped:
+ self._stream.writeln('Skipped {} tests'.format(len(skipped)))
+ self._stream.writeln('')
+ if self._options.verbose:
+ for test in skipped:
+ self._stream.writeln(' {}'.format(test))
+
+ self._print_tests_result_with_status(runner.STATUS_FAILED, runner)
+ self._print_tests_result_with_status(runner.STATUS_CRASHED, runner)
+ self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner)
+
+ for test, result in runner.results.iteritems():
+ status_to_string = {
+ runner.STATUS_FAILED: 'Failed',
+ runner.STATUS_CRASHED: 'Crashed',
+ runner.STATUS_TIMEOUT: 'Timedout',
+ }.get(result[0])
+ if not status_to_string:
+ continue
+ result_dictionary[status_to_string].append({'name': test, 'output': result[1]})
- skipped = []
- for test in test_names:
- if test not in runner.results:
- skipped.append(test)
- result_dictionary['Skipped'].append({'name': test, 'output': None})
- if skipped:
- self._stream.writeln('Skipped {} tests'.format(len(skipped)))
- self._stream.writeln('')
- if self._options.verbose:
- for test in skipped:
- self._stream.writeln(' {}'.format(test))
-
- self._print_tests_result_with_status(runner.STATUS_FAILED, runner)
- self._print_tests_result_with_status(runner.STATUS_CRASHED, runner)
- self._print_tests_result_with_status(runner.STATUS_TIMEOUT, runner)
-
- for test, result in runner.results.iteritems():
- status_to_string = {
- runner.STATUS_FAILED: 'Failed',
- runner.STATUS_CRASHED: 'Crashed',
- runner.STATUS_TIMEOUT: 'Timedout',
- }.get(result[0])
- if not status_to_string:
- continue
- result_dictionary[status_to_string].append({'name': test, 'output': result[1]})
+ if json_output:
+ self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
- if json_output:
- self.host.filesystem.write_text_file(json_output, json.dumps(result_dictionary, indent=4))
+ result = Manager.FAILED_TESTS
+
+ if self._options.report_urls:
+ self._stream.writeln('\n')
+ self._stream.write_update('Preparing upload data ...')
+
+ status_to_test_result = {
+ runner.STATUS_PASSED: None,
+ runner.STATUS_FAILED: Upload.Expectations.FAIL,
+ runner.STATUS_CRASHED: Upload.Expectations.CRASH,
+ runner.STATUS_TIMEOUT: Upload.Expectations.TIMEOUT,
+ }
+ upload = Upload(
+ suite='api-tests',
+ configuration=self._port.configuration_for_upload(self._port.target_host(0)),
+ details=Upload.create_details(options=self._options),
+ commits=self._port.commits_for_upload(),
+ run_stats=Upload.create_run_stats(
+ start_time=start_time,
+ end_time=end_time,
+ tests_skipped=len(result_dictionary['Skipped']),
+ ),
+ results={test: Upload.create_test_result(actual=status_to_test_result[result[0]])
+ for test, result in runner.results.iteritems() if result[0] in status_to_test_result},
+ )
+ for url in self._options.report_urls:
+ self._stream.write_update('Uploading to {} ...'.format(url))
+ if not upload.upload(url, log_line_func=self._stream.writeln):
+ result = Manager.FAILED_UPLOAD
+ self._stream.writeln('Uploads completed!')
- return Manager.FAILED_TESTS
+ return result
from webkitpy.common.host import Host
from webkitpy.layout_tests.views.metered_stream import MeteredStream
from webkitpy.port import configuration_options, platform_options, base, win
+from webkitpy.results.options import upload_options
EXCEPTIONAL_EXIT_STATUS = -1
INTERRUPT_EXIT_STATUS = -2
optparse.make_option('--force', action='store_true', default=False,
help='Run all tests, even DISABLED tests'),
]))
+ option_group_definitions.append(('Upload Options', upload_options()))
option_parser = optparse.OptionParser(
usage='run-api-tests [options] [<test names>...]',
from webkitpy.common import find_files
from webkitpy.common import read_checksum_from_png
+from webkitpy.common.checkout.scm.detection import SCMDetector
from webkitpy.common.memoized import memoized
from webkitpy.common.prettypatch import PrettyPatch
from webkitpy.common.system import path, pemfile
from webkitpy.layout_tests.servers import apache_http_server, http_server, http_server_base
from webkitpy.layout_tests.servers import web_platform_test_server
from webkitpy.layout_tests.servers import websocket_server
+from webkitpy.results.upload import Upload
_log = logging.getLogger(__name__)
# This is overridden by ports that need to do work in the parent process after a worker subprocess is spawned,
# such as closing file descriptors that were implicitly cloned to the worker.
pass
+
+ def configuration_for_upload(self, host=None):
+ configuration = self.test_configuration()
+ host = self.host or host
+
+ return Upload.create_configuration(
+ platform=host.platform.os_name,
+ version=str(host.platform.os_version),
+ version_name=host.platform.os_version_name(INTERNAL_TABLE) or host.platform.os_version_name(),
+ architecture=configuration.architecture,
+ style='guard-malloc' if self.get_option('guard_malloc') else configuration.build_type,
+ sdk=host.platform.build_version(),
+ )
+
+ @memoized
+ def commits_for_upload(self):
+ self.host.initialize_scm()
+
+ if port_config.apple_additions() and getattr(port_config.apple_additions(), 'repos', False):
+ repos = port_config.apple_additions().repos()
+ else:
+ repos = {}
+
+ up = os.path.dirname
+ repos['webkit'] = up(up(up(up(up(os.path.abspath(__file__))))))
+
+ commits = []
+ for repo_id, path in repos.iteritems():
+ scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(path)
+ commits.append(Upload.create_commit(
+ repository_id=repo_id,
+ id=scm.native_revision(path),
+ branch=scm.native_branch(path),
+ ))
+ return commits
def device_type(self):
return self.platform_device.device_type
+ @property
+ def build_version(self):
+ return self.platform_device.build_version
+
def __nonzero__(self):
return self.platform_device is not None
-# Copyright (C) 2018 Apple Inc. All rights reserved.
+# Copyright (C) 2018-2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
import logging
import traceback
+from webkitpy.common.version_name_map import VersionNameMap, PUBLIC_TABLE, INTERNAL_TABLE
from webkitpy.layout_tests.models.test_configuration import TestConfiguration
from webkitpy.port.darwin import DarwinPort
from webkitpy.port.simulator_process import SimulatorProcess
+from webkitpy.results.upload import Upload
from webkitpy.xcode.device_type import DeviceType
from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager
def device_version(self):
raise NotImplementedError
+
+ def configuration_for_upload(self, host=None):
+ configuration = self.test_configuration()
+
+ device_type = host.device_type if host else self.DEVICE_TYPE
+ model = device_type.hardware_family
+ if model and device_type.hardware_type:
+ model += ' {}'.format(device_type.hardware_type)
+
+ version = self.device_version()
+ version_name = None
+ for table in [INTERNAL_TABLE, PUBLIC_TABLE]:
+ version_name = VersionNameMap.map(self.host.platform).to_name(version, platform=device_type.software_variant.lower(), table=table)
+ if version_name:
+ break
+
+ return Upload.create_configuration(
+ platform=device_type.software_variant.lower(),
+ is_simulator=self.DEVICE_MANAGER == SimulatedDeviceManager,
+ version=str(version),
+ version_name=version_name,
+ architecture=configuration.architecture,
+ style='guard-malloc' if self.get_option('guard_malloc') else configuration.build_type,
+ model=model,
+ sdk=host.build_version if host else None,
+ )
-# Copyright (C) 2014-2016 Apple Inc. All rights reserved.
+# Copyright (C) 2014-2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
def test_max_child_processes(self):
port = self.make_port()
self.assertEqual(port.max_child_processes(DeviceType.from_string('Apple Watch')), 0)
+
+ def test_configuration_for_upload(self):
+ port = self.make_port()
+ self.assertEqual(
+ dict(
+ platform='ios',
+ is_simulator=True,
+ architecture='x86_64',
+ version='11',
+ version_name='iOS 11',
+ style='release',
+ ),
+ port.configuration_for_upload(),
+ )
# Copyright (C) 2011 Google Inc. All rights reserved.
-# Copyright (C) 2012, 2013, 2016 Apple Inc. All rights reserved.
+# Copyright (C) 2012-2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
worthless_patterns.append((re.compile('.*<<<< VMC >>>>.*\n'), ''))
worthless_patterns.append((re.compile('.*<<< FFR_Common >>>.*\n'), ''))
return worthless_patterns
+
+ def configuration_for_upload(self, host=None):
+ host = host or self.host
+ configuration = super(MacPort, self).configuration_for_upload(host=host)
+
+ output = host.executive.run_command(['/usr/sbin/sysctl', 'hw.model']).rstrip()
+ match = re.match(r'hw.model: (?P<model>.*)', output)
+ if match:
+ configuration['model'] = match.group('model')
+
+ return configuration
# Copyright (C) 2010 Google Inc. All rights reserved.
-# Copyright (C) 2014-2016 Apple Inc. All rights reserved.
+# Copyright (C) 2014-2019 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
port = self.make_port(options=MockOptions(webkit_test_runner=False), port_name='mac-wk2')
self.assertEqual(port.driver_name(), 'WebKitTestRunner')
+
+ def test_configuration_for_upload(self):
+ port = self.make_port()
+ self.assertEqual(
+ dict(
+ platform='mac',
+ is_simulator=False,
+ architecture='x86_64',
+ version='10.7',
+ version_name='Lion',
+ sdk='17A405',
+ style='release',
+ ),
+ port.configuration_for_upload(),
+ )
udid=device_info['udid'],
host=host,
device_type=device_type,
+ build_version=runtime.build_version,
))
SimulatedDeviceManager.AVAILABLE_DEVICES.append(result)
return result
'SHUTTING DOWN',
]
- def __init__(self, name, udid, host, device_type):
+ def __init__(self, name, udid, host, device_type, build_version):
assert device_type.software_version
self.name = name
self.udid = udid
self.device_type = device_type
+ self.build_version = build_version
self._state = SimulatedDevice.DeviceState.SHUTTING_DOWN
self._last_updated_state = time.time()
self.assertEquals(1, len(SimulatedDeviceManager.INITIALIZED_DEVICES))
self.assertEquals('34FB476C-6FA0-43C8-8945-1BD7A4EBF0DE', SimulatedDeviceManager.INITIALIZED_DEVICES[0].udid)
+ self.assertEquals('15A8401', SimulatedDeviceManager.INITIALIZED_DEVICES[0].build_version)
self.assertEquals(SimulatedDevice.DeviceState.BOOTED, SimulatedDeviceManager.INITIALIZED_DEVICES[0].platform_device.state())
SimulatedDeviceManager.tear_down(host)
git://git.webkit.org / WebKit-https.git / commitdiff