[ews-app] Update primary keys for handling multiple Buildbot instances
authoraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 1 Mar 2019 23:09:31 +0000 (23:09 +0000)
committeraakash_jain@apple.com <aakash_jain@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 1 Mar 2019 23:09:31 +0000 (23:09 +0000)
https://bugs.webkit.org/show_bug.cgi?id=195120

Reviewed by Stephanie Lewis.

Use a new primary key uid for build and step tables. Previous primary keys build_id and step_id
were not enough to handle multiple buildbot instances. This new primary key uid would be generated
by a combination of buildbot_instance_id and current primary key. e.g.: buildbot_instance_id + build_id

* BuildSlaveSupport/ews-app/ews/models/build.py: Added new primary key uid.
* BuildSlaveSupport/ews-app/ews/models/step.py: Ditto.
* BuildSlaveSupport/ews-app/ews/models/buildbotinstance.py: Generate uid and instance_id.
* BuildSlaveSupport/ews-app/ews/views/results.py: Updated to receive hostname in events.
* BuildSlaveSupport/ews-build/events.py: Send hostname along-with events.
* BuildSlaveSupport/ews-build/master.cfg: Ditto.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@242291 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/ews-app/ews/models/build.py
Tools/BuildSlaveSupport/ews-app/ews/models/buildbotinstance.py
Tools/BuildSlaveSupport/ews-app/ews/models/step.py
Tools/BuildSlaveSupport/ews-app/ews/views/results.py
Tools/BuildSlaveSupport/ews-build/events.py
Tools/BuildSlaveSupport/ews-build/master.cfg
Tools/ChangeLog

index 5718b55..0a310e1 100644 (file)
@@ -26,6 +26,7 @@ import logging
 
 from django.db import models
 from ews.config import ERR_UNEXPECTED, SUCCESS
+from ews.models.buildbotinstance import BuildbotInstance
 from ews.models.patch import Patch
 import ews.common.util as util
 
@@ -34,7 +35,7 @@ _log = logging.getLogger(__name__)
 
 class Build(models.Model):
     patch = models.ForeignKey(Patch, on_delete=models.CASCADE)
-    build_id = models.IntegerField(primary_key=True)
+    uid = models.TextField(primary_key=True)
     builder_id = models.IntegerField()
     builder_name = models.TextField()
     builder_display_name = models.TextField()
@@ -47,30 +48,31 @@ class Build(models.Model):
     modified = models.DateTimeField(auto_now=True)
 
     def __str__(self):
-        return str(self.build_id)
+        return str(self.uid)
 
     @classmethod
-    def save_build(cls, patch_id, build_id, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at=None):
+    def save_build(cls, patch_id, hostname, build_id, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at=None):
         if not Build.is_valid_result(patch_id, build_id, builder_id, number, result, state_string, started_at, complete_at):
             return ERR_UNEXPECTED
 
-        build = Build.get_existing_build(build_id)
+        uid = BuildbotInstance.get_uid(hostname, build_id)
+        build = Build.get_existing_build(uid)
         if build:
             # If the build data is already present in database, update it, e.g.: build complete event.
-            return Build.update_build(build, patch_id, build_id, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at)
+            return Build.update_build(build, patch_id, uid, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at)
 
         # Save the new build data, e.g.: build start event.
-        Build(patch_id, build_id, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at).save()
-        _log.info('Saved build {} in database for patch_id: {}'.format(build_id, patch_id))
+        Build(patch_id, uid, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at).save()
+        _log.info('Saved build {} in database for patch_id: {}'.format(uid, patch_id))
         return SUCCESS
 
     @classmethod
-    def update_build(cls, build, patch_id, build_id, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at):
+    def update_build(cls, build, patch_id, uid, builder_id, builder_name, builder_display_name, number, result, state_string, started_at, complete_at):
         if build.patch_id != patch_id:
             _log.error('patch_id {} does not match with patch_id {}. Ignoring new data.'.format(build.patch_id, patch_id))
             return ERR_UNEXPECTED
-        if build.build_id != build_id:
-            _log.error('build_id {} does not match with build_id {}. Ignoring new data.'.format(build.build_id, build_id))
+        if build.uid != uid:
+            _log.error('uid {} does not match with uid {}. Ignoring new data.'.format(build.uid, uid))
             return ERR_UNEXPECTED
         if build.builder_id != builder_id:
             _log.error('builder_id {} does not match with builder_id {}. Ignoring new data.'.format(build.builder_id, builder_id))
@@ -83,14 +85,14 @@ class Build(models.Model):
         build.started_at = started_at
         build.complete_at = complete_at
         build.save(update_fields=['result', 'state_string', 'started_at', 'complete_at', 'modified'])
-        _log.info('Updated build {} in database for patch_id: {}'.format(build_id, patch_id))
+        _log.info('Updated build {} in database for patch_id: {}'.format(uid, patch_id))
         return SUCCESS
 
     @classmethod
-    def get_existing_build(cls, build_id):
+    def get_existing_build(cls, uid):
         try:
-            return Build.objects.get(build_id=build_id)
-        except:
+            return Build.objects.get(uid=uid)
+        except Build.DoesNotExist:
             return None
 
     @classmethod
index 603b5b0..c7a1783 100644 (file)
 
 from __future__ import unicode_literals
 
+import logging
+
 from django.db import models
 
+_log = logging.getLogger(__name__)
+
 
 class BuildbotInstance(models.Model):
     instance_id = models.AutoField(primary_key=True)
@@ -34,3 +38,18 @@ class BuildbotInstance(models.Model):
 
     def __str__(self):
         return '{}_{}'.format(self.instance_id, self.hostname)
+
+    @classmethod
+    def get_instance_id(cls, hostname):
+        try:
+            return BuildbotInstance.objects.get(hostname=hostname, active=True).instance_id
+        except BuildbotInstance.DoesNotExist:
+            instance = BuildbotInstance(hostname=hostname)
+            instance.save()
+            _log.info('Created new buildbot instance id {} for hostname: {}'.format(instance.instance_id, hostname))
+            return instance.instance_id
+
+    @classmethod
+    def get_uid(cls, hostname, build_or_step_id):
+        instance_id = BuildbotInstance.get_instance_id(hostname)
+        return '{}_{}'.format(instance_id, build_or_step_id)
index 0812fee..9b380ac 100644 (file)
@@ -27,14 +27,15 @@ import logging
 from django.db import models
 from ews.config import ERR_UNEXPECTED, SUCCESS
 from ews.models.build import Build
+from ews.models.buildbotinstance import BuildbotInstance
 import ews.common.util as util
 
 _log = logging.getLogger(__name__)
 
 
 class Step(models.Model):
-    step_id = models.IntegerField(primary_key=True)
-    build = models.ForeignKey(Build, on_delete=models.CASCADE)
+    uid = models.TextField(primary_key=True)
+    build_uid = models.ForeignKey(Build, on_delete=models.CASCADE, db_column='build_uid')
     result = models.IntegerField(null=True, blank=True)
     state_string = models.TextField()
     started_at = models.IntegerField(null=True, blank=True)
@@ -43,44 +44,44 @@ class Step(models.Model):
     modified = models.DateTimeField(auto_now=True)
 
     def __str__(self):
-        return str(self.step_id)
+        return str(self.uid)
 
     @classmethod
-    def save_step(cls, step_id, build_id, result, state_string, started_at, complete_at=None):
+    def save_step(cls, hostname, step_id, build_id, result, state_string, started_at, complete_at=None):
         if not Step.is_valid_result(step_id, build_id, result, state_string, started_at, complete_at):
             return ERR_UNEXPECTED
 
-        step = Step.get_existing_step(step_id)
+        step_uid = BuildbotInstance.get_uid(hostname, step_id)
+        build_uid = BuildbotInstance.get_uid(hostname, build_id)
+
+        step = Step.get_existing_step(step_uid)
         if step:
             # If the step data is already present in database, update it, e.g.: step complete event.
-            return Step.update_step(step, step_id, build_id, result, state_string, started_at, complete_at)
+            return Step.update_step(step, step_uid, build_uid, result, state_string, started_at, complete_at)
 
         # Save the new step data, e.g.: step start event.
-        Step(step_id, build_id, result, state_string, started_at, complete_at).save()
-        _log.debug('Saved step {} in database for build: {}'.format(step_id, build_id))
+        Step(step_uid, build_uid, result, state_string, started_at, complete_at).save()
+        _log.debug('Saved step {} in database for build: {}'.format(step_uid, build_uid))
         return SUCCESS
 
     @classmethod
-    def update_step(cls, step, step_id, build_id, result, state_string, started_at, complete_at):
-        if step.step_id != step_id:
-            _log.error('step_id {} does not match with step_id {}. Ignoring new data.'.format(step.step_id, step_id))
-            return ERR_UNEXPECTED
-        if step.build_id != build_id:
-            _log.error('build_id {} does not match with build_id {}. Ignoring new data.'.format(step.build_id, build_id))
+    def update_step(cls, step, step_uid, build_uid, result, state_string, started_at, complete_at):
+        if step.uid != step_uid:
+            _log.error('step_uid {} does not match with step_uid {}. Ignoring new data.'.format(step.uid, step_uid))
             return ERR_UNEXPECTED
         step.result = result
         step.state_string = state_string
         step.started_at = started_at
         step.complete_at = complete_at
         step.save(update_fields=['result', 'state_string', 'started_at', 'complete_at', 'modified'])
-        _log.info('Updated step {} in database for build: {}'.format(step_id, build_id))
+        _log.info('Updated step {} in database for build: {}'.format(step_uid, build_uid))
         return SUCCESS
 
     @classmethod
-    def get_existing_step(cls, step_id):
+    def get_existing_step(cls, uid):
         try:
-            return Step.objects.get(step_id=step_id)
-        except:
+            return Step.objects.get(uid=uid)
+        except Step.DoesNotExist:
             return None
 
     @classmethod
index de0249b..6e76731 100644 (file)
@@ -59,7 +59,7 @@ class Results(View):
         if not patch_id or patch_id < 1:
             return HttpResponse("Invalid patch id: {}.".format(patch_id))
 
-        Build.save_build(patch_id=int(patch_id), build_id=data['build_id'], builder_id=data['builder_id'], builder_name=data['builder_name'],
+        Build.save_build(patch_id=int(patch_id), hostname=data['hostname'], build_id=data['build_id'], builder_id=data['builder_id'], builder_name=data['builder_name'],
                    builder_display_name=data['builder_display_name'], number=data['number'], result=data['result'],
                    state_string=data['state_string'], started_at=data['started_at'], complete_at=data['complete_at'])
         return HttpResponse("Saved data for patch: {}.\n".format(patch_id))
@@ -68,7 +68,7 @@ class Results(View):
         if not self.is_valid_result(data):
             return HttpResponse("Incomplete data.")
 
-        Step.save_step(step_id=data['step_id'], build_id=data['build_id'], result=data['result'],
+        Step.save_step(hostname=data['hostname'], step_id=data['step_id'], build_id=data['build_id'], result=data['result'],
                    state_string=data['state_string'], started_at=data['started_at'], complete_at=data['complete_at'])
         return HttpResponse("Saved data for step: {}.\n".format(data['step_id']))
 
@@ -77,9 +77,9 @@ class Results(View):
             _log.error("Invalid data type: {}".format(data['type']))
             return False
 
-        required_keys = {u'ews-build': ['patch_id', 'build_id', 'builder_id', 'builder_name', 'builder_display_name',
+        required_keys = {u'ews-build': ['hostname', 'patch_id', 'build_id', 'builder_id', 'builder_name', 'builder_display_name',
                                            'number', 'result', 'state_string', 'started_at', 'complete_at'],
-                         u'ews-step': ['step_id', 'build_id', 'result', 'state_string', 'started_at', 'complete_at']}
+                         u'ews-step': ['hostname', 'step_id', 'build_id', 'result', 'state_string', 'started_at', 'complete_at']}
 
         for key in required_keys.get(data.get('type')):
             if key not in data:
index 709df42..d715d7e 100644 (file)
@@ -73,7 +73,7 @@ class Events(service.BuildbotService):
 
     EVENT_SERVER_ENDPOINT = 'http://ews.webkit-uat.org/results/'
 
-    def __init__(self, type_prefix='', name='Events'):
+    def __init__(self, master_hostname, type_prefix='', name='Events'):
         """
         Initialize the Events Plugin. Sends data to event server on specific buildbot events.
         :param type_prefix: [optional] prefix we want to add to the 'type' field on the json we send
@@ -85,6 +85,7 @@ class Events(service.BuildbotService):
         if type_prefix and not type_prefix.endswith("-"):
             type_prefix += "-"
         self.type_prefix = type_prefix
+        self.master_hostname = master_hostname
 
     def sendData(self, data):
         agent = Agent(reactor)
@@ -115,6 +116,7 @@ class Events(service.BuildbotService):
         data = {
             "type": self.type_prefix + "build",
             "status": "started",
+            "hostname": self.master_hostname,
             "patch_id": self.getPatchID(build),
             "build_id": build.get('buildid'),
             "builder_id": build.get('builderid'),
@@ -142,6 +144,7 @@ class Events(service.BuildbotService):
         data = {
             "type": self.type_prefix + "build",
             "status": "finished",
+            "hostname": self.master_hostname,
             "patch_id": self.getPatchID(build),
             "build_id": build.get('buildid'),
             "builder_id": build.get('builderid'),
@@ -161,6 +164,7 @@ class Events(service.BuildbotService):
         data = {
             "type": self.type_prefix + "step",
             "status": "started",
+            "hostname": self.master_hostname,
             "step_id": step.get('stepid'),
             "build_id": step.get('buildid'),
             "result": step.get('results'),
@@ -175,6 +179,7 @@ class Events(service.BuildbotService):
         data = {
             "type": self.type_prefix + "step",
             "status": "finished",
+            "hostname": self.master_hostname,
             "step_id": step.get('stepid'),
             "build_id": step.get('buildid'),
             "result": step.get('results'),
index 5ec0d03..b30802e 100644 (file)
@@ -26,5 +26,5 @@ c['buildbotNetUsageData'] = None
 
 loadConfig.loadBuilderConfig(c, use_localhost_worker=is_test_mode_enabled)
 
-event_reporter = Events(type_prefix='ews')
+event_reporter = Events(master_hostname='ews-build.webkit.org', type_prefix='ews')
 c['services'] = [event_reporter]
index 93448b5..46b389d 100644 (file)
@@ -1,3 +1,21 @@
+2019-03-01  Aakash Jain  <aakash_jain@apple.com>
+
+        [ews-app] Update primary keys for handling multiple Buildbot instances
+        https://bugs.webkit.org/show_bug.cgi?id=195120
+
+        Reviewed by Stephanie Lewis.
+
+        Use a new primary key uid for build and step tables. Previous primary keys build_id and step_id
+        were not enough to handle multiple buildbot instances. This new primary key uid would be generated
+        by a combination of buildbot_instance_id and current primary key. e.g.: buildbot_instance_id + build_id
+
+        * BuildSlaveSupport/ews-app/ews/models/build.py: Added new primary key uid.
+        * BuildSlaveSupport/ews-app/ews/models/step.py: Ditto.
+        * BuildSlaveSupport/ews-app/ews/models/buildbotinstance.py: Generate uid and instance_id.
+        * BuildSlaveSupport/ews-app/ews/views/results.py: Updated to receive hostname in events.
+        * BuildSlaveSupport/ews-build/events.py: Send hostname along-with events.
+        * BuildSlaveSupport/ews-build/master.cfg: Ditto.
+
 2019-03-01  Don Olmstead  <don.olmstead@sony.com>
 
         [WinCairo] Enable service worker