upload.py gets confused by git-svn checkouts, unable to upload test results from...
authorbburg@apple.com <bburg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 9 Mar 2020 21:50:24 +0000 (21:50 +0000)
committerbburg@apple.com <bburg@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 9 Mar 2020 21:50:24 +0000 (21:50 +0000)
https://bugs.webkit.org/show_bug.cgi?id=208729
<rdar://problem/60105447>

Reviewed by Jonathan Bedard.

Improve upload.py to look for the svn branch and revision for the 'webkit'
project and prefer those to the git equivalents (eg, prefer trunk/rNNNNN to master/<hash>).

* Scripts/webkitpy/common/checkout/scm/git.py:
(Git):
(Git.git_svn_id_regexp):
(Git._field_from_git_svn_id):
(Git.svn_revision):
(Git.svn_branch):
(Git.svn_url):
(Git.native_branch):
Refactor svn revision, branch, url to use the same regexp and helper code.

* Scripts/webkitpy/common/checkout/scm/stub_repository.py:
(StubRepository.svn_branch):
Expose a stub since this is used unconditionally by Port.commits_for_upload.

* Scripts/webkitpy/common/checkout/scm/svn.py:
(SVN.svn_branch):
(SVN.native_revision):
(SVN):
(SVN.native_branch):
Refactor so the actual implementation is contained in svn_revision / svn_branch.
The native_* methods call out to the SVN equivalents since this is an SVN repository.

* Scripts/webkitpy/port/base.py:
(Port.commits_for_upload):
Add special git-svn handling for the WebKit repository so that SVN branch and revision
is preferred over the git equivalents.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@258164 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/common/checkout/scm/git.py
Tools/Scripts/webkitpy/common/checkout/scm/scm_unittest.py
Tools/Scripts/webkitpy/common/checkout/scm/stub_repository.py
Tools/Scripts/webkitpy/common/checkout/scm/svn.py
Tools/Scripts/webkitpy/port/base.py

index 979ec0a..9d669ea 100644 (file)
@@ -1,3 +1,41 @@
+2020-03-06  Brian Burg  <bburg@apple.com>
+
+        upload.py gets confused by git-svn checkouts, unable to upload test results from my desk build
+        https://bugs.webkit.org/show_bug.cgi?id=208729
+        <rdar://problem/60105447>
+
+        Reviewed by Jonathan Bedard.
+
+        Improve upload.py to look for the svn branch and revision for the 'webkit'
+        project and prefer those to the git equivalents (eg, prefer trunk/rNNNNN to master/<hash>).
+
+        * Scripts/webkitpy/common/checkout/scm/git.py:
+        (Git):
+        (Git.git_svn_id_regexp):
+        (Git._field_from_git_svn_id):
+        (Git.svn_revision):
+        (Git.svn_branch):
+        (Git.svn_url):
+        (Git.native_branch):
+        Refactor svn revision, branch, url to use the same regexp and helper code.
+
+        * Scripts/webkitpy/common/checkout/scm/stub_repository.py:
+        (StubRepository.svn_branch):
+        Expose a stub since this is used unconditionally by Port.commits_for_upload.
+
+        * Scripts/webkitpy/common/checkout/scm/svn.py:
+        (SVN.svn_branch):
+        (SVN.native_revision):
+        (SVN):
+        (SVN.native_branch):
+        Refactor so the actual implementation is contained in svn_revision / svn_branch.
+        The native_* methods call out to the SVN equivalents since this is an SVN repository.
+
+        * Scripts/webkitpy/port/base.py:
+        (Port.commits_for_upload):
+        Add special git-svn handling for the WebKit repository so that SVN branch and revision
+        is preferred over the git equivalents.
+
 2020-03-09  Antoine Quint  <graouts@apple.com>
 
         TestWebKitAPI fails to build on watchOS
index 617fc2f..cd924f1 100644 (file)
@@ -29,6 +29,7 @@
 
 import datetime
 import logging
+import os
 import re
 
 from webkitpy.common.memoized import memoized
@@ -55,6 +56,7 @@ class Git(SCM, SVNRepository):
     # Git doesn't appear to document error codes, but seems to return
     # 1 or 128, mostly.
     ERROR_FILE_IS_MISSING = 128
+    GIT_SVN_ID_REGEXP = r"^\s*git-svn-id:\s(?P<svn_url>(?P<base_url>.*)/(branch)?(?P<svn_branch>(.*)))@(?P<svn_revision>\d+)\ (?P<svn_uuid>[a-fA-F0-9-]+)$"
 
     executable_name = 'git'
 
@@ -281,12 +283,26 @@ class Git(SCM, SVNRepository):
     def _most_recent_log_for_revision(self, revision, path):
         return self._run_git(['log', '-1', revision, '--date=iso', self.find_checkout_root(path)])
 
-    def svn_revision(self, path):
+    def _field_from_git_svn_id(self, path, field):
+        # Keep this in sync with the regex from git_svn_id_regexp() above.
+        allowed_fields = ['svn_url', 'base_url', 'svn_branch', 'svn_revision', 'svn_uuid']
+        if field not in allowed_fields:
+            raise ValueError("Unsupported field for git-svn-id: " + field)
+
         git_log = self._most_recent_log_matching('git-svn-id:', path)
-        match = re.search("^\s*git-svn-id:.*@(?P<svn_revision>\d+)\ ", git_log, re.MULTILINE)
+        match = re.search(self.GIT_SVN_ID_REGEXP, git_log, re.MULTILINE)
         if not match:
             return ""
-        return str(match.group('svn_revision'))
+        return str(match.group(field))
+
+    def svn_revision(self, path):
+        return self._field_from_git_svn_id(path, 'svn_revision')
+
+    def svn_branch(self, path):
+        return self._field_from_git_svn_id(path, 'svn_branch')
+
+    def svn_url(self, path):
+        return self._field_from_git_svn_id(path, 'svn_url')
 
     def native_revision(self, path):
         return self._run_git(['-C', self.find_checkout_root(path), 'log', '-1', '--pretty=format:%H'])
@@ -299,14 +315,6 @@ class Git(SCM, SVNRepository):
             return result[6:]
         return result
 
-    def svn_url(self):
-        git_command = ['svn', 'info']
-        status = self._run_git(git_command)
-        match = re.search(r'^URL: (?P<url>.*)$', status, re.MULTILINE)
-        if not match:
-            return ""
-        return match.group('url')
-
     def timestamp_of_revision(self, path, revision):
         git_log = self._most_recent_log_matching('git-svn-id:.*@%s' % revision, path)
         match = re.search("^Date:\s*(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$", git_log, re.MULTILINE)
index 325333f..47d9cc3 100644 (file)
@@ -564,6 +564,11 @@ OcmYex&reD$;sO8*F9L)B
     def _shared_test_head_svn_revision(self):
         self.assertEqual(self.scm.head_svn_revision(), '5')
 
+    def _shared_test_svn_revision(self, scm):
+        self.assertEqual(scm.svn_revision(scm.checkout_root), '5')
+
+    def _shared_test_svn_branch(self, scm):
+        self.assertEqual(scm.svn_branch(scm.checkout_root), 'trunk')
 
 # Context manager that overrides the current timezone.
 class TimezoneOverride(object):
@@ -920,6 +925,12 @@ END
     def test_head_svn_revision(self):
         self._shared_test_head_svn_revision()
 
+    def test_svn_revision(self):
+        self._shared_test_svn_revision(self.scm)
+
+    def test_svn_branch(self):
+        self._shared_test_svn_branch(self.scm)
+
     def test_native_revision(self):
         self.assertEqual(self.scm.head_svn_revision(), self.scm.native_revision('.'))
         self.assertEqual(self.scm.native_revision('.'), '5')
@@ -1749,3 +1760,16 @@ MOCK run_command: ['git', 'log', '-1', '--grep=git-svn-id:', '--date=iso', './MO
 
         scm._run_git = lambda args: '1360317321'
         self.assertEqual(scm.timestamp_of_native_revision('some-path', '1a1c3b08814bc2a8c15b0f6db63cdce68f2aaa7a'), '2013-02-08T09:55:21Z')
+
+    def test_svn_data_from_git_svn_id(self):
+        scm = self.make_scm()
+        scm.find_checkout_root = lambda path: ''
+        scm._most_recent_log_matching = lambda grep_str, path: 'git-svn-id: http://svn.webkit.org/repository/webkit/trunk@258024 268f45cc-cd09-0410-ab3c-d52691b4dbfc'
+        self.assertEqual(scm.svn_revision(scm.checkout_root), '258024')
+        self.assertEqual(scm.svn_branch(scm.checkout_root), 'trunk')
+        self.assertEqual(scm.svn_url(scm.checkout_root), 'http://svn.webkit.org/repository/webkit/trunk')
+
+        scm._most_recent_log_matching = lambda grep_str, path: 'git-svn-id: http://svn.webkit.org/repository/webkit/branch/specialSubmission@258000 268f45cc-cd09-0410-ab3c-d52691b4dbfc'
+        self.assertEqual(scm.svn_revision(scm.checkout_root), '258000')
+        self.assertEqual(scm.svn_branch(scm.checkout_root), 'specialSubmission')
+        self.assertEqual(scm.svn_url(scm.checkout_root), 'http://svn.webkit.org/repository/webkit/branch/specialSubmission')
index ce49e1a..f42fbed 100644 (file)
@@ -63,6 +63,9 @@ class StubRepository(SCM):
     def svn_revision(self, path):
         return self.native_revision(path)
 
+    def svn_branch(self, path):
+        return self.native_branch(path)
+
     def native_revision(self, path):
         return self._find_parent_path_matching_callback_condition(path, lambda path: self._decode_json(path)['id'], filesystem=self._filesystem)
 
index 10ff742..319bf8c 100644 (file)
@@ -276,10 +276,7 @@ class SVN(SCM, SVNRepository):
     def svn_revision(self, path):
         return self.value_from_svn_info(path, 'Revision')
 
-    def native_revision(self, path):
-        return self.svn_revision(path)
-
-    def native_branch(self, path):
+    def svn_branch(self, path):
         relative_url = self.value_from_svn_info(path, 'Relative URL')[2:]
         if relative_url.startswith('trunk'):
             return 'trunk'
@@ -287,6 +284,12 @@ class SVN(SCM, SVNRepository):
             return relative_url.split('/')[1]
         raise Exception('{} is not a branch'.format(relative_url.split('/')[0]))
 
+    def native_revision(self, path):
+        return self.svn_revision(path)
+
+    def native_branch(self, path):
+        return self.svn_branch(path)
+
     def timestamp_of_revision(self, path, revision):
         # We use --xml to get timestamps like 2013-02-08T08:18:04.964409Z
         repository_root = self.value_from_svn_info(self.checkout_root, 'Repository Root')
index 07fa418..e1a19ce 100644 (file)
@@ -1606,9 +1606,23 @@ class Port(object):
         commits = []
         for repo_id, path in repos.items():
             scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(path)
+
+            # If using git-svn for WebKit, prefer the SVN branch/revision.
+            svn_revision = scm.svn_revision(path)
+            if repo_id == 'webkit' and svn_revision:
+                used_revision = svn_revision
+            else:
+                used_revision = scm.native_revision(path)
+
+            svn_branch = scm.svn_branch(path)
+            if repo_id == 'webkit' and svn_branch:
+                used_branch = svn_branch
+            else:
+                used_branch = scm.native_branch(path)
+
             commits.append(Upload.create_commit(
                 repository_id=repo_id,
-                id=scm.native_revision(path),
-                branch=scm.native_branch(path),
+                id=used_revision,
+                branch=used_branch,
             ))
         return commits