perf-o-matic: Extract logic to generate JSON responses as classes to unit test
[WebKit-https.git] / Websites / webkit-perf.appspot.com / models_unittest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import models
31 import unittest
32
33 from datetime import datetime
34 from google.appengine.api import memcache
35 from google.appengine.ext import testbed
36 from time import mktime
37
38
39 class DataStoreTestsBase(unittest.TestCase):
40     def setUp(self):
41         self.testbed = testbed.Testbed()
42         self.testbed.activate()
43         self.testbed.init_datastore_v3_stub()
44
45     def tearDown(self):
46         self.testbed.deactivate()
47
48     def assertThereIsNoInstanceOf(self, model):
49         self.assertEqual(len(model.all().fetch(5)), 0)
50
51     def assertOnlyInstance(self, only_instasnce):
52         self.assertEqual(len(only_instasnce.__class__.all().fetch(5)), 1)
53         self.assertTrue(only_instasnce.__class__.get(only_instasnce.key()))
54
55     def assertEqualUnorderedList(self, list1, list2):
56         self.assertEqual(set(list1), set(list2))
57
58
59 class HelperTests(DataStoreTestsBase):
60     def _assert_there_is_exactly_one_id_holder_and_matches(self, id):
61         id_holders = models.NumericIdHolder.all().fetch(5)
62         self.assertEqual(len(id_holders), 1)
63         self.assertTrue(id_holders[0])
64         self.assertEqual(id_holders[0].key().id(), id)
65
66     def test_create_in_transaction_with_numeric_id_holder(self):
67
68         def execute(id):
69             return models.Branch(id=id, name='some branch', key_name='some-branch').put()
70
71         self.assertThereIsNoInstanceOf(models.Branch)
72         self.assertThereIsNoInstanceOf(models.NumericIdHolder)
73
74         self.assertTrue(models.create_in_transaction_with_numeric_id_holder(execute))
75
76         branches = models.Branch.all().fetch(5)
77         self.assertEqual(len(branches), 1)
78         self.assertEqual(branches[0].name, 'some branch')
79         self.assertEqual(branches[0].key().name(), 'some-branch')
80
81         self._assert_there_is_exactly_one_id_holder_and_matches(branches[0].id)
82
83     def test_failing_in_create_in_transaction_with_numeric_id_holder(self):
84
85         def execute(id):
86             return None
87
88         self.assertThereIsNoInstanceOf(models.Branch)
89         self.assertThereIsNoInstanceOf(models.NumericIdHolder)
90
91         self.assertFalse(models.create_in_transaction_with_numeric_id_holder(execute))
92
93         self.assertThereIsNoInstanceOf(models.Branch)
94         self.assertThereIsNoInstanceOf(models.NumericIdHolder)
95
96     def test_raising_in_create_in_transaction_with_numeric_id_holder(self):
97
98         def execute(id):
99             raise TypeError
100             return None
101
102         self.assertThereIsNoInstanceOf(models.Branch)
103         self.assertThereIsNoInstanceOf(models.NumericIdHolder)
104
105         self.assertRaises(TypeError, models.create_in_transaction_with_numeric_id_holder, (execute))
106
107         self.assertThereIsNoInstanceOf(models.Branch)
108         self.assertThereIsNoInstanceOf(models.NumericIdHolder)
109
110     def test_delete_model_with_numeric_id_holder(self):
111
112         def execute(id):
113             return models.Branch(id=id, name='some branch', key_name='some-branch').put()
114
115         branch = models.Branch.get(models.create_in_transaction_with_numeric_id_holder(execute))
116         self.assertOnlyInstance(branch)
117
118         models.delete_model_with_numeric_id_holder(branch)
119
120         self.assertThereIsNoInstanceOf(models.Branch)
121         self.assertThereIsNoInstanceOf(models.NumericIdHolder)
122
123     def test_model_from_numeric_id(self):
124
125         def execute(id):
126             return models.Branch(id=id, name='some branch', key_name='some-branch').put()
127
128         branch = models.Branch.get(models.create_in_transaction_with_numeric_id_holder(execute))
129
130         self.assertEqual(models.model_from_numeric_id(branch.id, models.Branch).key(), branch.key())
131         self.assertEqual(models.model_from_numeric_id(branch.id + 1, models.Branch), None)
132         models.delete_model_with_numeric_id_holder(branch)
133         self.assertEqual(models.model_from_numeric_id(branch.id, models.Branch), None)
134
135
136 class BranchTests(DataStoreTestsBase):
137     def test_create_if_possible(self):
138         self.assertThereIsNoInstanceOf(models.Branch)
139
140         branch = models.Branch.create_if_possible('some-branch', 'some branch')
141         self.assertTrue(branch)
142         self.assertTrue(branch.key().name(), 'some-branch')
143         self.assertTrue(branch.name, 'some branch')
144         self.assertOnlyInstance(branch)
145
146         self.assertFalse(models.Branch.create_if_possible('some-branch', 'some other branch'))
147         self.assertTrue(branch.name, 'some branch')
148         self.assertOnlyInstance(branch)
149
150
151 class PlatformTests(DataStoreTestsBase):
152     def test_create_if_possible(self):
153         self.assertThereIsNoInstanceOf(models.Platform)
154
155         platform = models.Platform.create_if_possible('some-platform', 'some platform')
156         self.assertTrue(platform)
157         self.assertTrue(platform.key().name(), 'some-platform')
158         self.assertTrue(platform.name, 'some platform')
159         self.assertOnlyInstance(platform)
160
161         self.assertFalse(models.Platform.create_if_possible('some-platform', 'some other platform'))
162         self.assertTrue(platform.name, 'some platform')
163         self.assertOnlyInstance(platform)
164
165
166 class BuilderTests(DataStoreTestsBase):
167     def test_create(self):
168         builder_key = models.Builder.create('some builder', 'some password')
169         self.assertTrue(builder_key)
170         builder = models.Builder.get(builder_key)
171         self.assertEqual(builder.key().name(), 'some builder')
172         self.assertEqual(builder.name, 'some builder')
173         self.assertEqual(builder.password, models.Builder._hashed_password('some password'))
174
175     def test_update_password(self):
176         builder = models.Builder.get(models.Builder.create('some builder', 'some password'))
177         self.assertEqual(builder.password, models.Builder._hashed_password('some password'))
178         builder.update_password('other password')
179         self.assertEqual(builder.password, models.Builder._hashed_password('other password'))
180
181         # Make sure it's saved
182         builder = models.Builder.get(builder.key())
183         self.assertEqual(builder.password, models.Builder._hashed_password('other password'))
184
185     def test_hashed_password(self):
186         self.assertNotEqual(models.Builder._hashed_password('some password'), 'some password')
187         self.assertFalse('some password' in models.Builder._hashed_password('some password'))
188         self.assertEqual(len(models.Builder._hashed_password('some password')), 64)
189
190     def test_authenticate(self):
191         builder = models.Builder.get(models.Builder.create('some builder', 'some password'))
192         self.assertTrue(builder.authenticate('some password'))
193         self.assertFalse(builder.authenticate('bad password'))
194
195
196 def _create_some_builder():
197     branch = models.Branch.create_if_possible('some-branch', 'Some Branch')
198     platform = models.Platform.create_if_possible('some-platform', 'Some Platform')
199     builder_key = models.Builder.create('some-builder', 'Some Builder')
200     return branch, platform, models.Builder.get(builder_key)
201
202
203 class BuildTests(DataStoreTestsBase):
204     def test_get_or_insert_from_log(self):
205         branch, platform, builder = _create_some_builder()
206
207         timestamp = datetime.now().replace(microsecond=0)
208         log = models.ReportLog(timestamp=timestamp, headers='some headers',
209             payload='{"branch": "some-branch", "platform": "some-platform", "builder-name": "some-builder",' +
210                 '"build-number": 123, "webkit-revision": 456, "timestamp": %d}' % int(mktime(timestamp.timetuple())))
211
212         self.assertThereIsNoInstanceOf(models.Build)
213
214         build = models.Build.get_or_insert_from_log(log)
215         self.assertTrue(build)
216         self.assertEqual(build.branch.key(), branch.key())
217         self.assertEqual(build.platform.key(), platform.key())
218         self.assertEqual(build.builder.key(), builder.key())
219         self.assertEqual(build.buildNumber, 123)
220         self.assertEqual(build.revision, 456)
221         self.assertEqual(build.chromiumRevision, None)
222         self.assertEqual(build.timestamp, timestamp)
223
224         self.assertOnlyInstance(build)
225
226
227 class TestModelTests(DataStoreTestsBase):
228     def test_update_or_insert(self):
229         branch = models.Branch.create_if_possible('some-branch', 'Some Branch')
230         platform = models.Platform.create_if_possible('some-platform', 'Some Platform')
231
232         self.assertThereIsNoInstanceOf(models.Test)
233
234         test = models.Test.update_or_insert('some-test', branch, platform)
235         self.assertTrue(test)
236         self.assertEqual(test.branches, [branch.key()])
237         self.assertEqual(test.platforms, [platform.key()])
238         self.assertOnlyInstance(test)
239
240     def test_update_or_insert_to_update(self):
241         branch = models.Branch.create_if_possible('some-branch', 'Some Branch')
242         platform = models.Platform.create_if_possible('some-platform', 'Some Platform')
243         test = models.Test.update_or_insert('some-test', branch, platform)
244         self.assertOnlyInstance(test)
245
246         other_branch = models.Branch.create_if_possible('other-branch', 'Other Branch')
247         other_platform = models.Platform.create_if_possible('other-platform', 'Other Platform')
248         test = models.Test.update_or_insert('some-test', other_branch, other_platform)
249         self.assertOnlyInstance(test)
250         self.assertEqualUnorderedList(test.branches, [branch.key(), other_branch.key()])
251         self.assertEqualUnorderedList(test.platforms, [platform.key(), other_platform.key()])
252
253         test = models.Test.get(test.key())
254         self.assertEqualUnorderedList(test.branches, [branch.key(), other_branch.key()])
255         self.assertEqualUnorderedList(test.platforms, [platform.key(), other_platform.key()])
256
257
258 class TestResultTests(DataStoreTestsBase):
259     def _create_build(self):
260         branch, platform, builder = _create_some_builder()
261         build_key = models.Build(key_name='some-build', branch=branch, platform=platform, builder=builder,
262             buildNumber=1, revision=100, timestamp=datetime.now()).put()
263         return models.Build.get(build_key)
264
265     def test_get_or_insert_value(self):
266         build = self._create_build()
267         self.assertThereIsNoInstanceOf(models.TestResult)
268         result = models.TestResult.get_or_insert_from_parsed_json('some-test', build, 50)
269         self.assertOnlyInstance(result)
270         self.assertEqual(result.name, 'some-test')
271         self.assertEqual(result.build.key(), build.key())
272         self.assertEqual(result.value, 50.0)
273         self.assertEqual(result.valueMedian, None)
274         self.assertEqual(result.valueStdev, None)
275         self.assertEqual(result.valueMin, None)
276         self.assertEqual(result.valueMax, None)
277
278     def test_get_or_insert_stat_value(self):
279         build = self._create_build()
280         self.assertThereIsNoInstanceOf(models.TestResult)
281         result = models.TestResult.get_or_insert_from_parsed_json('some-test', build,
282             {"avg": 40, "median": "40.1", "stdev": 3.25, "min": 30.5, "max": 45})
283         self.assertOnlyInstance(result)
284         self.assertEqual(result.name, 'some-test')
285         self.assertEqual(result.build.key(), build.key())
286         self.assertEqual(result.value, 40.0)
287         self.assertEqual(result.valueMedian, 40.1)
288         self.assertEqual(result.valueStdev, 3.25)
289         self.assertEqual(result.valueMin, 30.5)
290         self.assertEqual(result.valueMax, 45)
291
292
293 class ReportLogTests(DataStoreTestsBase):
294     def _create_log_with_payload(self, payload):
295         return models.ReportLog(timestamp=datetime.now(), headers='some headers', payload=payload)
296
297     def test_parsed_payload(self):
298         log = self._create_log_with_payload('')
299         self.assertFalse('_parsed' in log.__dict__)
300         self.assertEqual(log._parsed_payload(), False)
301         self.assertEqual(log._parsed, False)
302
303         log = self._create_log_with_payload('{"key": "value", "another key": 1}')
304         self.assertEqual(log._parsed_payload(), {"key": "value", "another key": 1})
305         self.assertEqual(log._parsed, {"key": "value", "another key": 1})
306
307     def test_get_value(self):
308         log = self._create_log_with_payload('{"string": "value", "integer": 1, "float": 1.1}')
309         self.assertEqual(log.get_value('string'), 'value')
310         self.assertEqual(log.get_value('integer'), 1)
311         self.assertEqual(log.get_value('float'), 1.1)
312         self.assertEqual(log.get_value('bad'), None)
313
314     def test_results(self):
315         log = self._create_log_with_payload('{"results": 123}')
316         self.assertEqual(log.results(), 123)
317
318         log = self._create_log_with_payload('{"key": "value"}')
319         self.assertEqual(log.results(), None)
320
321     def test_builder(self):
322         log = self._create_log_with_payload('{"key": "value"}')
323         self.assertEqual(log.builder(), None)
324
325         builder_name = "Chromium Mac Release (Perf)"
326         log = self._create_log_with_payload('{"builder-name": "%s"}' % builder_name)
327         self.assertEqual(log.builder(), None)
328
329         builder_key = models.Builder.create(builder_name, 'some password')
330         log = self._create_log_with_payload('{"builder-name": "%s"}' % builder_name)
331         self.assertEqual(log.builder().key(), builder_key)
332
333     def test_branch(self):
334         log = self._create_log_with_payload('{"key": "value"}')
335         self.assertEqual(log.branch(), None)
336
337         log = self._create_log_with_payload('{"branch": "some-branch"}')
338         self.assertEqual(log.branch(), None)
339
340         branch = models.Branch.create_if_possible("some-branch", "Some Branch")
341         log = self._create_log_with_payload('{"branch": "some-branch"}')
342         self.assertEqual(log.branch().key(), branch.key())
343
344     def test_platform(self):
345         log = self._create_log_with_payload('{"key": "value"}')
346         self.assertEqual(log.platform(), None)
347
348         log = self._create_log_with_payload('{"platform": "some-platform"}')
349         self.assertEqual(log.platform(), None)
350
351         platform = models.Platform.create_if_possible("some-platform", "Some Platform")
352         log = self._create_log_with_payload('{"platform": "some-platform"}')
353         self.assertEqual(log.platform().key(), platform.key())
354
355     def test_build_number(self):
356         log = self._create_log_with_payload('{"build-number": 123}')
357         self.assertEqual(log.build_number(), 123)
358
359         log = self._create_log_with_payload('{"key": "value"}')
360         self.assertEqual(log.build_number(), None)
361
362     def test_webkit_revision(self):
363         log = self._create_log_with_payload('{"key": "value"}')
364         self.assertEqual(log.webkit_revision(), None)
365
366         log = self._create_log_with_payload('{"webkit-revision": 123}')
367         self.assertEqual(log.webkit_revision(), 123)
368
369     def chromium_revision(self):
370         log = self._create_log_with_payload('{"chromium-revision": 123}')
371         self.assertEqual(log.webkit_revision(), 123)
372
373         log = self._create_log_with_payload('{"key": "value"}')
374         self.assertEqual(log.webkit_revision(), None)
375
376
377 class PersistentCacheTests(DataStoreTestsBase):
378     def setUp(self):
379         self.testbed = testbed.Testbed()
380         self.testbed.activate()
381         self.testbed.init_datastore_v3_stub()
382         self.testbed.init_memcache_stub()
383
384     def _assert_persistent_cache(self, name, value):
385         self.assertEqual(models.PersistentCache.get_by_key_name(name).value, value)
386         self.assertEqual(memcache.get(name), value)
387
388     def test_set(self):
389         self.assertThereIsNoInstanceOf(models.PersistentCache)
390
391         models.PersistentCache.set_cache('some-cache', 'some data')
392         self._assert_persistent_cache('some-cache', 'some data')
393
394         models.PersistentCache.set_cache('some-cache', 'some other data')
395
396         self._assert_persistent_cache('some-cache', 'some other data')
397
398     def test_get(self):
399         self.assertEqual(memcache.get('some-cache'), None)
400         self.assertEqual(models.PersistentCache.get_cache('some-cache'), None)
401
402         models.PersistentCache.set_cache('some-cache', 'some data')
403
404         self.assertEqual(memcache.get('some-cache'), 'some data')
405         self.assertEqual(models.PersistentCache.get_cache('some-cache'), 'some data')
406
407         memcache.delete('some-cache')
408         self.assertEqual(memcache.get('some-cache'), None)
409         self.assertEqual(models.PersistentCache.get_cache('some-cache'), 'some data')
410
411
412 if __name__ == '__main__':
413     unittest.main()