Handle DRT crash caused by Android OOM.

https://bugs.webkit.org/show_bug.cgi?id=82310

Patch by Hao Zheng <zhenghao@chromium.org> on 2012-03-27
Reviewed by Tony Chang.

When Android is OOM, it sends a SIGKILL (137) signal to DRT. DRT
is stopped silently and regarded as crashed. Re-run the test for
such crash.

Add some debug log to NRWT.

* Scripts/webkitpy/layout_tests/port/chromium.py:
(ChromiumDriver.run_test):
* Scripts/webkitpy/layout_tests/port/chromium_android.py:
(ChromiumAndroidPort.__init__):
(ChromiumAndroidPort.get_last_stacktrace):
(ChromiumAndroidDriver.__init__):
(ChromiumAndroidDriver._start):
(ChromiumAndroidDriver.run_test):
(ChromiumAndroidDriver._get_drt_return_value):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@112359 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index 250ef63..3ee736a 100644 (file)
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,26 @@
+2012-03-27  Hao Zheng  <zhenghao@chromium.org>
+
+        Handle DRT crash caused by Android OOM.
+        https://bugs.webkit.org/show_bug.cgi?id=82310
+
+        Reviewed by Tony Chang.
+
+        When Android is OOM, it sends a SIGKILL (137) signal to DRT. DRT
+        is stopped silently and regarded as crashed. Re-run the test for
+        such crash.
+
+        Add some debug log to NRWT.
+
+        * Scripts/webkitpy/layout_tests/port/chromium.py:
+        (ChromiumDriver.run_test):
+        * Scripts/webkitpy/layout_tests/port/chromium_android.py:
+        (ChromiumAndroidPort.__init__):
+        (ChromiumAndroidPort.get_last_stacktrace):
+        (ChromiumAndroidDriver.__init__):
+        (ChromiumAndroidDriver._start):
+        (ChromiumAndroidDriver.run_test):
+        (ChromiumAndroidDriver._get_drt_return_value):
+
 2012-03-27  Ojan Vafai  <ojan@chromium.org>
 
         Fix expected results for some unittest failures.

diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
index ada4a81..bca1db8 100755 (executable)
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
@@ -576,6 +576,8 @@ class ChromiumDriver(Driver):
 
             line, crash = self._write_command_and_read_line(input=None)
 
+        if crash and line is not None:
+            error.append(line)
         run_time = time.time() - start_time
         output_image = self._output_image_with_retry()
 

diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py
index 61a253a..7e9ca19 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py
@@ -27,6 +27,9 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import logging
+import re
+import signal
+import time
 
 from webkitpy.layout_tests.port import chromium
 from webkitpy.layout_tests.port import factory
@@ -152,6 +155,7 @@ class ChromiumAndroidPort(chromium.ChromiumPort):
         adb_args = self.get_option('adb_args')
         if adb_args:
             self._adb_command += shlex.split(adb_args)
+        self._drt_retry_after_killed = 0
 
     def default_test_timeout_ms(self):
         # Android platform has less computing power than desktop platforms.
@@ -361,9 +365,10 @@ class ChromiumAndroidPort(chromium.ChromiumPort):
 
     def get_last_stacktrace(self):
         tombstones = self._run_adb_command(['shell', 'ls', '-n', '/data/tombstones'])
-        tombstones = tombstones.rstrip().split('\n')
         if not tombstones:
+            _log.error('DRT crashed, but no tombstone found!')
             return ''
+        tombstones = tombstones.rstrip().split('\n')
         last_tombstone = tombstones[0].split()
         for tombstone in tombstones[1:]:
             # Format of fields:
@@ -397,6 +402,7 @@ class ChromiumAndroidDriver(chromium.ChromiumDriver):
     def __init__(self, port, worker_number, pixel_tests, no_timeout=False):
         chromium.ChromiumDriver.__init__(self, port, worker_number, pixel_tests, no_timeout)
         self._device_image_path = None
+        self._drt_return_parser = re.compile('#DRT_RETURN (\d+)')
 
     def _start(self, pixel_tests, per_test_args):
         # Convert the original command line into to two parts:
@@ -440,7 +446,7 @@ class ChromiumAndroidDriver(chromium.ChromiumDriver):
             # the process. Sleep 1 second (long enough for debuggerd to dump
             # stack) before exiting the shell to ensure the process has quit,
             # otherwise the exit will fail because "You have stopped jobs".
-            drt_cmd = '%s %s 2>%s;sleep 1;exit\n' % (DEVICE_DRT_PATH, ' '.join(drt_args), DEVICE_DRT_STDERR)
+            drt_cmd = '%s %s 2>%s;echo "#DRT_RETURN $?";sleep 1;exit\n' % (DEVICE_DRT_PATH, ' '.join(drt_args), DEVICE_DRT_STDERR)
             _log.debug('Starting DumpRenderTree: ' + drt_cmd)
 
             # Wait until DRT echos '#READY'.
@@ -467,13 +473,30 @@ class ChromiumAndroidDriver(chromium.ChromiumDriver):
 
     def run_test(self, driver_input):
         driver_output = chromium.ChromiumDriver.run_test(self, driver_input)
+
+        drt_return = self._get_drt_return_value(driver_output.error)
+        if drt_return is not None:
+            _log.debug('DumpRenderTree return value: %d' % drt_return)
         # FIXME: Retrieve stderr from the target.
         if driver_output.crash:
+            # When Android is OOM, it sends a SIGKILL signal to DRT. DRT
+            # is stopped silently and regarded as crashed. Re-run the test for
+            # such crash.
+            if drt_return == 128 + signal.SIGKILL:
+                self._port._drt_retry_after_killed += 1
+                if self._port._drt_retry_after_killed > 10:
+                    raise AssertionError('DumpRenderTree is killed by Android for too many times!')
+                _log.error('DumpRenderTree is killed by SIGKILL. Retry the test (%d).' % self._port._drt_retry_after_killed)
+                self.stop()
+                # Sleep 10 seconds to let system recover.
+                time.sleep(10)
+                return self.run_test(driver_input)
             # Fetch the stack trace from the tombstone file.
             # FIXME: sometimes the crash doesn't really happen so that no
             # tombstone is generated. In that case we fetch the wrong stack
             # trace.
-            driver_output.error += self._port.get_last_stacktrace()
+            driver_output.error += self._port.get_last_stacktrace().encode('ascii', 'ignore')
+            driver_output.error += self._port._run_adb_command(['logcat', '-d']).encode('ascii', 'ignore')
         return driver_output
 
     def stop(self):
@@ -523,6 +546,10 @@ class ChromiumAndroidDriver(chromium.ChromiumDriver):
         # We use the Shell output as a crash hint.
         return line is not None and line.find('[1] + Stopped (signal)') >= 0
 
+    def _get_drt_return_value(self, error):
+        return_match = self._drt_return_parser.search(error)
+        return None if (return_match is None) else int(return_match.group(1))
+
     def _read_prompt(self):
         last_char = ''
         while True: