Perf-o-matic should store "values" and support array'ed input
[WebKit-https.git] / Websites / webkit-perf.appspot.com / models_unittest.py
1 #!/usr/bin/env python
2 # Copyright (C) 2012 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import json
31 import models
32 import unittest
33
34 from datetime import datetime
35 from datetime import timedelta
36 from google.appengine.api import memcache
37 from google.appengine.ext import testbed
38 from time import mktime
39
40 from models import NumericIdHolder
41 from models import Branch
42 from models import Platform
43 from models import Builder
44 from models import Build
45 from models import Test
46 from models import TestResult
47 from models import ReportLog
48 from models import PersistentCache
49 from models import Runs
50 from models import DashboardImage
51 from models import create_in_transaction_with_numeric_id_holder
52 from models import delete_model_with_numeric_id_holder
53 from models import model_from_numeric_id
54
55
56 class DataStoreTestsBase(unittest.TestCase):
57     def setUp(self):
58         self.testbed = testbed.Testbed()
59         self.testbed.activate()
60         self.testbed.init_datastore_v3_stub()
61
62     def tearDown(self):
63         self.testbed.deactivate()
64
65     def assertThereIsNoInstanceOf(self, model):
66         self.assertEqual(len(model.all().fetch(5)), 0)
67
68     def assertOnlyInstance(self, only_instasnce):
69         self.assertOnlyInstances([only_instasnce])
70
71     def assertOnlyInstances(self, expected_instances):
72         actual_instances = expected_instances[0].__class__.all().fetch(len(expected_instances) + 1)
73         self.assertEqualUnorderedModelList(actual_instances, expected_instances)
74
75     def assertEqualUnorderedModelList(self, list1, list2):
76         self.assertEqualUnorderedList([item.key() for item in list1], [item.key() for item in list1])
77
78     def assertEqualUnorderedList(self, list1, list2):
79         self.assertEqual(set(list1), set(list2))
80         self.assertEqual(len(list1), len(list2))
81
82
83 class HelperTests(DataStoreTestsBase):
84     def _assert_there_is_exactly_one_id_holder_and_matches(self, id):
85         id_holders = NumericIdHolder.all().fetch(5)
86         self.assertEqual(len(id_holders), 1)
87         self.assertTrue(id_holders[0])
88         self.assertEqual(id_holders[0].key().id(), id)
89
90     def test_create_in_transaction_with_numeric_id_holder(self):
91
92         def execute(id):
93             return Branch(id=id, name='some branch', key_name='some-branch').put()
94
95         self.assertThereIsNoInstanceOf(Branch)
96         self.assertThereIsNoInstanceOf(NumericIdHolder)
97
98         self.assertTrue(create_in_transaction_with_numeric_id_holder(execute))
99
100         branches = Branch.all().fetch(5)
101         self.assertEqual(len(branches), 1)
102         self.assertEqual(branches[0].name, 'some branch')
103         self.assertEqual(branches[0].key().name(), 'some-branch')
104
105         self._assert_there_is_exactly_one_id_holder_and_matches(branches[0].id)
106
107     def test_failing_in_create_in_transaction_with_numeric_id_holder(self):
108
109         def execute(id):
110             return None
111
112         self.assertThereIsNoInstanceOf(Branch)
113         self.assertThereIsNoInstanceOf(NumericIdHolder)
114
115         self.assertFalse(create_in_transaction_with_numeric_id_holder(execute))
116
117         self.assertThereIsNoInstanceOf(Branch)
118         self.assertThereIsNoInstanceOf(NumericIdHolder)
119
120     def test_raising_in_create_in_transaction_with_numeric_id_holder(self):
121
122         def execute(id):
123             raise TypeError
124             return None
125
126         self.assertThereIsNoInstanceOf(Branch)
127         self.assertThereIsNoInstanceOf(NumericIdHolder)
128
129         self.assertRaises(TypeError, create_in_transaction_with_numeric_id_holder, (execute))
130
131         self.assertThereIsNoInstanceOf(Branch)
132         self.assertThereIsNoInstanceOf(NumericIdHolder)
133
134     def test_delete_model_with_numeric_id_holder(self):
135
136         def execute(id):
137             return Branch(id=id, name='some branch', key_name='some-branch').put()
138
139         branch = Branch.get(create_in_transaction_with_numeric_id_holder(execute))
140         self.assertOnlyInstance(branch)
141
142         delete_model_with_numeric_id_holder(branch)
143
144         self.assertThereIsNoInstanceOf(Branch)
145         self.assertThereIsNoInstanceOf(NumericIdHolder)
146
147     def test_model_from_numeric_id(self):
148
149         def execute(id):
150             return Branch(id=id, name='some branch', key_name='some-branch').put()
151
152         branch = Branch.get(create_in_transaction_with_numeric_id_holder(execute))
153
154         self.assertEqual(model_from_numeric_id(branch.id, Branch).key(), branch.key())
155         self.assertEqual(model_from_numeric_id(branch.id + 1, Branch), None)
156         delete_model_with_numeric_id_holder(branch)
157         self.assertEqual(model_from_numeric_id(branch.id, Branch), None)
158
159
160 class BranchTests(DataStoreTestsBase):
161     def test_create_if_possible(self):
162         self.assertThereIsNoInstanceOf(Branch)
163
164         branch = Branch.create_if_possible('some-branch', 'some branch')
165         self.assertTrue(branch)
166         self.assertTrue(branch.key().name(), 'some-branch')
167         self.assertTrue(branch.name, 'some branch')
168         self.assertOnlyInstance(branch)
169
170         self.assertFalse(Branch.create_if_possible('some-branch', 'some other branch'))
171         self.assertTrue(branch.name, 'some branch')
172         self.assertOnlyInstance(branch)
173
174
175 class PlatformTests(DataStoreTestsBase):
176     def test_create_if_possible(self):
177         self.assertThereIsNoInstanceOf(Platform)
178
179         platform = Platform.create_if_possible('some-platform', 'some platform')
180         self.assertTrue(platform)
181         self.assertTrue(platform.key().name(), 'some-platform')
182         self.assertTrue(platform.name, 'some platform')
183         self.assertOnlyInstance(platform)
184
185         self.assertFalse(Platform.create_if_possible('some-platform', 'some other platform'))
186         self.assertTrue(platform.name, 'some platform')
187         self.assertOnlyInstance(platform)
188
189
190 class BuilderTests(DataStoreTestsBase):
191     def test_create(self):
192         builder_key = Builder.create('some builder', 'some password')
193         self.assertTrue(builder_key)
194         builder = Builder.get(builder_key)
195         self.assertEqual(builder.key().name(), 'some builder')
196         self.assertEqual(builder.name, 'some builder')
197         self.assertEqual(builder.password, Builder._hashed_password('some password'))
198
199     def test_update_password(self):
200         builder = Builder.get(Builder.create('some builder', 'some password'))
201         self.assertEqual(builder.password, Builder._hashed_password('some password'))
202         builder.update_password('other password')
203         self.assertEqual(builder.password, Builder._hashed_password('other password'))
204
205         # Make sure it's saved
206         builder = Builder.get(builder.key())
207         self.assertEqual(builder.password, Builder._hashed_password('other password'))
208
209     def test_hashed_password(self):
210         self.assertNotEqual(Builder._hashed_password('some password'), 'some password')
211         self.assertFalse('some password' in Builder._hashed_password('some password'))
212         self.assertEqual(len(Builder._hashed_password('some password')), 64)
213
214     def test_authenticate(self):
215         builder = Builder.get(Builder.create('some builder', 'some password'))
216         self.assertTrue(builder.authenticate('some password'))
217         self.assertFalse(builder.authenticate('bad password'))
218
219
220 def _create_some_builder():
221     branch = Branch.create_if_possible('some-branch', 'Some Branch')
222     platform = Platform.create_if_possible('some-platform', 'Some Platform')
223     builder_key = Builder.create('some-builder', 'Some Builder')
224     return branch, platform, Builder.get(builder_key)
225
226
227 def _create_build(branch, platform, builder, key_name='some-build'):
228     build_key = Build(key_name=key_name, branch=branch, platform=platform, builder=builder,
229         buildNumber=1, revision=100, timestamp=datetime.now()).put()
230     return Build.get(build_key)
231
232
233 class BuildTests(DataStoreTestsBase):
234     def test_get_or_insert_from_log(self):
235         branch, platform, builder = _create_some_builder()
236
237         timestamp = datetime.now().replace(microsecond=0)
238         log = ReportLog(timestamp=timestamp, headers='some headers',
239             payload='{"branch": "some-branch", "platform": "some-platform", "builder-name": "some-builder",' +
240                 '"build-number": 123, "webkit-revision": 456, "timestamp": %d}' % int(mktime(timestamp.timetuple())))
241
242         self.assertThereIsNoInstanceOf(Build)
243
244         build = Build.get_or_insert_from_log(log)
245         self.assertTrue(build)
246         self.assertEqual(build.branch.key(), branch.key())
247         self.assertEqual(build.platform.key(), platform.key())
248         self.assertEqual(build.builder.key(), builder.key())
249         self.assertEqual(build.buildNumber, 123)
250         self.assertEqual(build.revision, 456)
251         self.assertEqual(build.chromiumRevision, None)
252         self.assertEqual(build.timestamp, timestamp)
253
254         self.assertOnlyInstance(build)
255
256
257 class TestModelTests(DataStoreTestsBase):
258     def test_update_or_insert(self):
259         branch = Branch.create_if_possible('some-branch', 'Some Branch')
260         platform = Platform.create_if_possible('some-platform', 'Some Platform')
261
262         self.assertThereIsNoInstanceOf(Test)
263
264         test = Test.update_or_insert('some-test', branch, platform)
265         self.assertTrue(test)
266         self.assertEqual(test.branches, [branch.key()])
267         self.assertEqual(test.platforms, [platform.key()])
268         self.assertEqual(test.unit, None)
269         self.assertOnlyInstance(test)
270
271     def test_update_or_insert_with_unit(self):
272         branch = Branch.create_if_possible('some-branch', 'Some Branch')
273         platform = Platform.create_if_possible('some-platform', 'Some Platform')
274         test = Test.update_or_insert('some-test', branch, platform, 'runs/s')
275         self.assertOnlyInstance(test)
276         self.assertEqualUnorderedList(test.unit, 'runs/s')
277
278     def test_update_or_insert_to_update(self):
279         branch = Branch.create_if_possible('some-branch', 'Some Branch')
280         platform = Platform.create_if_possible('some-platform', 'Some Platform')
281         test = Test.update_or_insert('some-test', branch, platform)
282         self.assertOnlyInstance(test)
283
284         other_branch = Branch.create_if_possible('other-branch', 'Other Branch')
285         other_platform = Platform.create_if_possible('other-platform', 'Other Platform')
286         test = Test.update_or_insert('some-test', other_branch, other_platform, 'ms')
287         self.assertOnlyInstance(test)
288         self.assertEqualUnorderedList(test.branches, [branch.key(), other_branch.key()])
289         self.assertEqualUnorderedList(test.platforms, [platform.key(), other_platform.key()])
290         self.assertEqualUnorderedList(test.unit, 'ms')
291
292     def test_merge(self):
293         branch, platform, builder = _create_some_builder()
294         some_build = _create_build(branch, platform, builder)
295         some_result = TestResult.get_or_insert_from_parsed_json('some-test', some_build, 50)
296         some_test = Test.update_or_insert('some-test', branch, platform)
297
298         other_build = _create_build(branch, platform, builder, 'other-build')
299         other_result = TestResult.get_or_insert_from_parsed_json('other-test', other_build, 30)
300         other_test = Test.update_or_insert('other-test', branch, platform)
301
302         self.assertOnlyInstances([some_result, other_result])
303         self.assertNotEqual(some_result.key(), other_result.key())
304         self.assertOnlyInstances([some_test, other_test])
305
306         self.assertRaises(AssertionError, some_test.merge, (some_test))
307         self.assertOnlyInstances([some_test, other_test])
308
309         some_test.merge(other_test)
310         results_for_some_test = TestResult.all()
311         results_for_some_test.filter('name =', 'some-test')
312         results_for_some_test = results_for_some_test.fetch(5)
313         self.assertEqual(len(results_for_some_test), 2)
314
315         self.assertEqual(results_for_some_test[0].name, 'some-test')
316         self.assertEqual(results_for_some_test[1].name, 'some-test')
317
318         if results_for_some_test[0].value == 50:
319             self.assertEqual(results_for_some_test[1].value, 30)
320         else:
321             self.assertEqual(results_for_some_test[1].value, 50)
322
323
324 class TestResultTests(DataStoreTestsBase):
325     def test_get_or_insert_value(self):
326         branch, platform, builder = _create_some_builder()
327         build = _create_build(branch, platform, builder)
328         self.assertThereIsNoInstanceOf(TestResult)
329         result = TestResult.get_or_insert_from_parsed_json('some-test', build, 50)
330         self.assertOnlyInstance(result)
331         self.assertEqual(result.name, 'some-test')
332         self.assertEqual(result.build.key(), build.key())
333         self.assertEqual(result.value, 50.0)
334         self.assertEqual(result.valueMedian, None)
335         self.assertEqual(result.valueStdev, None)
336         self.assertEqual(result.valueMin, None)
337         self.assertEqual(result.valueMax, None)
338
339     def test_get_or_insert_stat_value(self):
340         branch, platform, builder = _create_some_builder()
341         build = _create_build(branch, platform, builder)
342         self.assertThereIsNoInstanceOf(TestResult)
343         result = TestResult.get_or_insert_from_parsed_json('some-test', build,
344             {"avg": 40, "median": "40.1", "stdev": 3.25, "min": 30.5, "max": 45})
345         self.assertOnlyInstance(result)
346         self.assertEqual(result.name, 'some-test')
347         self.assertEqual(result.build.key(), build.key())
348         self.assertEqual(result.value, 40.0)
349         self.assertEqual(result.valueMedian, 40.1)
350         self.assertEqual(result.valueStdev, 3.25)
351         self.assertEqual(result.valueMin, 30.5)
352         self.assertEqual(result.valueMax, 45)
353         self.assertEqual(result.values, [])
354
355     def test_get_or_insert_stat_value_with_values(self):
356         branch, platform, builder = _create_some_builder()
357         build = _create_build(branch, platform, builder)
358         result = TestResult.get_or_insert_from_parsed_json('some-test', build,
359             {"avg": 40, "median": "40.1", "stdev": 3.25, "min": 30.5, "max": 45, "values": [1.0, 2.0, 3.0]})
360         self.assertEqual(result.values, [1.0, 2.0, 3.0])
361
362     def test_replace_to_change_test_name(self):
363         branch, platform, builder = _create_some_builder()
364         build = _create_build(branch, platform, builder)
365         self.assertThereIsNoInstanceOf(TestResult)
366         result = TestResult.get_or_insert_from_parsed_json('some-test', build, 50)
367         self.assertOnlyInstance(result)
368         self.assertEqual(result.name, 'some-test')
369
370         new_result = result.replace_to_change_test_name('other-test')
371         self.assertNotEqual(result, new_result)
372         self.assertOnlyInstance(new_result)
373
374         self.assertEqual(new_result.name, 'other-test')
375         self.assertEqual(new_result.build.key(), result.build.key())
376         self.assertEqual(new_result.value, result.value)
377         self.assertEqual(new_result.valueMedian, None)
378         self.assertEqual(new_result.valueStdev, None)
379         self.assertEqual(new_result.valueMin, None)
380         self.assertEqual(new_result.valueMax, None)
381
382     def test_replace_to_change_test_name_with_stat_value(self):
383         branch, platform, builder = _create_some_builder()
384         build = _create_build(branch, platform, builder)
385         self.assertThereIsNoInstanceOf(TestResult)
386         result = TestResult.get_or_insert_from_parsed_json('some-test', build,
387             {"avg": 40, "median": "40.1", "stdev": 3.25, "min": 30.5, "max": 45})
388         self.assertOnlyInstance(result)
389         self.assertEqual(result.name, 'some-test')
390
391         new_result = result.replace_to_change_test_name('other-test')
392         self.assertNotEqual(result, new_result)
393         self.assertOnlyInstance(new_result)
394
395         self.assertEqual(new_result.name, 'other-test')
396         self.assertEqual(new_result.build.key(), result.build.key())
397         self.assertEqual(new_result.value, result.value)
398         self.assertEqual(result.value, 40.0)
399         self.assertEqual(result.valueMedian, 40.1)
400         self.assertEqual(result.valueStdev, 3.25)
401         self.assertEqual(result.valueMin, 30.5)
402         self.assertEqual(result.valueMax, 45)
403
404     def test_replace_to_change_test_name_overrides_conflicting_result(self):
405         branch, platform, builder = _create_some_builder()
406         build = _create_build(branch, platform, builder)
407         self.assertThereIsNoInstanceOf(TestResult)
408         result = TestResult.get_or_insert_from_parsed_json('some-test', build, 20)
409         self.assertOnlyInstance(result)
410
411         conflicting_result = TestResult.get_or_insert_from_parsed_json('other-test', build, 10)
412
413         new_result = result.replace_to_change_test_name('other-test')
414         self.assertNotEqual(result, new_result)
415         self.assertOnlyInstance(new_result)
416
417         self.assertEqual(new_result.name, 'other-test')
418         self.assertEqual(TestResult.get(conflicting_result.key()).value, 20)
419
420
421 class ReportLogTests(DataStoreTestsBase):
422     def _create_log_with_payload(self, payload):
423         return ReportLog(timestamp=datetime.now(), headers='some headers', payload=payload)
424
425     def test_parsed_payload(self):
426         log = self._create_log_with_payload('')
427         self.assertFalse('_parsed' in log.__dict__)
428         self.assertEqual(log._parsed_payload(), False)
429         self.assertEqual(log._parsed, False)
430
431         log = self._create_log_with_payload('{"key": "value", "another key": 1}')
432         self.assertEqual(log._parsed_payload(), {"key": "value", "another key": 1})
433         self.assertEqual(log._parsed, {"key": "value", "another key": 1})
434
435     def test_get_value(self):
436         log = self._create_log_with_payload('{"string": "value", "integer": 1, "float": 1.1}')
437         self.assertEqual(log.get_value('string'), 'value')
438         self.assertEqual(log.get_value('integer'), 1)
439         self.assertEqual(log.get_value('float'), 1.1)
440         self.assertEqual(log.get_value('bad'), None)
441
442     def test_results(self):
443         log = self._create_log_with_payload('{"results": 123}')
444         self.assertEqual(log.results(), 123)
445
446         log = self._create_log_with_payload('{"key": "value"}')
447         self.assertEqual(log.results(), None)
448
449     def test_results_are_well_formed(self):
450
451         def assert_results_are_well_formed(json, expected):
452             self.assertEqual(self._create_log_with_payload(json).results_are_well_formed(), expected)
453
454         assert_results_are_well_formed('{"results": 123}', False)
455         assert_results_are_well_formed('{"results": {"test": 123}}', True)
456         assert_results_are_well_formed('{"results": {"test": 123, "other-test": 456}}', True)
457         assert_results_are_well_formed('{"results": {"test": 123, "other-test": 456, "bad-test": "hi"}}', False)
458         assert_results_are_well_formed('{"results": {"test": {"avg": 456}}}', True)
459         assert_results_are_well_formed('{"results": {"test": {"avg": 456, "median": "hello"}}}', False)
460         assert_results_are_well_formed('{"results": {"test": {"avg": 456, "median": 789}}}', True)
461         assert_results_are_well_formed('{"results": {"test": {"avg": 456, "unit": "bytes"}}}', True)
462         assert_results_are_well_formed('{"results": {"test": {"avg": 456, "unit": "bytes", "values": [1.0, 2.0, 3.0]}}}', True)
463
464         assert_results_are_well_formed('[]', False)
465         assert_results_are_well_formed('[{"results": {"test": 123}}]', True)
466         assert_results_are_well_formed('[{"results": {"test": 123}}, {"results": {"test": 123}}]', False)
467         assert_results_are_well_formed('[{"results": {"test": {"avg": 456, "unit": "bytes", "values": [1.0, 2.0, 3.0]}}}]', True)
468
469     def test_builder(self):
470         log = self._create_log_with_payload('{"key": "value"}')
471         self.assertEqual(log.builder(), None)
472
473         builder_name = "Chromium Mac Release (Perf)"
474         log = self._create_log_with_payload('{"builder-name": "%s"}' % builder_name)
475         self.assertEqual(log.builder(), None)
476
477         builder_key = Builder.create(builder_name, 'some password')
478         log = self._create_log_with_payload('{"builder-name": "%s"}' % builder_name)
479         self.assertEqual(log.builder().key(), builder_key)
480
481     def test_branch(self):
482         log = self._create_log_with_payload('{"key": "value"}')
483         self.assertEqual(log.branch(), None)
484
485         log = self._create_log_with_payload('{"branch": "some-branch"}')
486         self.assertEqual(log.branch(), None)
487
488         branch = Branch.create_if_possible("some-branch", "Some Branch")
489         log = self._create_log_with_payload('{"branch": "some-branch"}')
490         self.assertEqual(log.branch().key(), branch.key())
491
492     def test_platform(self):
493         log = self._create_log_with_payload('{"key": "value"}')
494         self.assertEqual(log.platform(), None)
495
496         log = self._create_log_with_payload('{"platform": "some-platform"}')
497         self.assertEqual(log.platform(), None)
498
499         platform = Platform.create_if_possible("some-platform", "Some Platform")
500         log = self._create_log_with_payload('{"platform": "some-platform"}')
501         self.assertEqual(log.platform().key(), platform.key())
502
503     def test_build_number(self):
504         log = self._create_log_with_payload('{"build-number": 123}')
505         self.assertEqual(log.build_number(), 123)
506
507         log = self._create_log_with_payload('{"key": "value"}')
508         self.assertEqual(log.build_number(), None)
509
510     def test_webkit_revision(self):
511         log = self._create_log_with_payload('{"key": "value"}')
512         self.assertEqual(log.webkit_revision(), None)
513
514         log = self._create_log_with_payload('{"webkit-revision": 123}')
515         self.assertEqual(log.webkit_revision(), 123)
516
517     def test_chromium_revision(self):
518         log = self._create_log_with_payload('{"chromium-revision": 123}')
519         self.assertEqual(log.chromium_revision(), 123)
520
521         log = self._create_log_with_payload('{"key": "value"}')
522         self.assertEqual(log.chromium_revision(), None)
523
524     def test_results_in_array(self):
525         platform = Platform.create_if_possible("some-platform", "Some Platform")
526         log = self._create_log_with_payload('[{"platform": "some-platform", "build-number": 123, "webkit-revision": 456}]')
527         self.assertEqual(log.platform().key(), platform.key())
528         self.assertEqual(log.build_number(), 123)
529         self.assertEqual(log.webkit_revision(), 456)
530
531
532 class PersistentCacheTests(DataStoreTestsBase):
533     def setUp(self):
534         super(PersistentCacheTests, self).setUp()
535         self.testbed.init_memcache_stub()
536
537     def _assert_persistent_cache(self, name, value):
538         self.assertEqual(PersistentCache.get_by_key_name(name).value, value)
539         self.assertEqual(memcache.get(name), value)
540
541     def test_set_cache(self):
542         self.assertThereIsNoInstanceOf(PersistentCache)
543
544         PersistentCache.set_cache('some-cache', 'some data')
545         self._assert_persistent_cache('some-cache', 'some data')
546
547         PersistentCache.set_cache('some-cache', 'some other data')
548
549         self._assert_persistent_cache('some-cache', 'some other data')
550
551     def test_get_cache(self):
552         self.assertEqual(memcache.get('some-cache'), None)
553         self.assertEqual(PersistentCache.get_cache('some-cache'), None)
554
555         PersistentCache.set_cache('some-cache', 'some data')
556
557         self.assertEqual(memcache.get('some-cache'), 'some data')
558         self.assertEqual(PersistentCache.get_cache('some-cache'), 'some data')
559
560         memcache.delete('some-cache')
561         self.assertEqual(memcache.get('some-cache'), None)
562         self.assertEqual(PersistentCache.get_cache('some-cache'), 'some data')
563
564
565 class RunsTest(DataStoreTestsBase):
566     def setUp(self):
567         super(RunsTest, self).setUp()
568         self.testbed.init_memcache_stub()
569
570     def _create_results(self, branch, platform, builder, test_name, values, timestamps=None, starting_revision=100):
571         builds = []
572         results = []
573         for i, value in enumerate(values):
574             build = Build(branch=branch, platform=platform, builder=builder,
575                 buildNumber=i, revision=starting_revision + i, timestamp=timestamps[i] if timestamps else datetime.now())
576             build.put()
577             result = TestResult(name=test_name, build=build, value=value)
578             result.put()
579             builds.append(build)
580             results.append(result)
581         return builds, results
582
583     def test_generate_runs(self):
584         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
585         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
586         some_builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
587         some_test = Test.update_or_insert('some-test', some_branch, some_platform)
588
589         builds, results = self._create_results(some_branch, some_platform, some_builder, 'some-test', [50.0, 51.0, 52.0, 49.0, 48.0])
590         last_i = 0
591         for i, (build, result) in enumerate(Runs._generate_runs(some_branch, some_platform, some_test)):
592             self.assertEqual(build.buildNumber, i)
593             self.assertEqual(build.revision, 100 + i)
594             self.assertEqual(result.name, 'some-test')
595             self.assertEqual(result.value, results[i].value)
596             last_i = i
597         self.assertTrue(last_i + 1, len(results))
598
599     def test_update_or_insert(self):
600         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
601         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
602         some_builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
603         some_test = Test.update_or_insert('some-test', some_branch, some_platform)
604         self.assertThereIsNoInstanceOf(Runs)
605
606         runs = Runs.update_or_insert(some_branch, some_platform, some_test)
607         self.assertOnlyInstance(runs)
608         self.assertEqual(runs.json_runs, '')
609         self.assertEqual(runs.json_averages, '')
610         self.assertEqual(runs.json_min, None)
611         self.assertEqual(runs.json_max, None)
612         old_memcache_value = memcache.get(Runs._key_name(some_branch.id, some_platform.id, some_test.id))
613         self.assertTrue(old_memcache_value)
614
615         runs.delete()
616         self.assertThereIsNoInstanceOf(Runs)
617
618         builds, results = self._create_results(some_branch, some_platform, some_builder, 'some-test', [50.0])
619         runs = Runs.update_or_insert(some_branch, some_platform, some_test)
620         self.assertOnlyInstance(runs)
621         self.assertTrue(runs.json_runs.startswith('[5, [4, 0, 100, null],'))
622         self.assertEqual(json.loads('{' + runs.json_averages + '}'), {"100": 50.0})
623         self.assertEqual(runs.json_min, 50.0)
624         self.assertEqual(runs.json_max, 50.0)
625         self.assertNotEqual(memcache.get(Runs._key_name(some_branch.id, some_platform.id, some_test.id)), old_memcache_value)
626
627     def test_update_incrementally(self):
628         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
629         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
630         some_builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
631         some_test = Test.update_or_insert('some-test', some_branch, some_platform)
632         self.assertThereIsNoInstanceOf(Runs)
633
634         timestamps = [datetime.now(), datetime.now()]
635         builds, results = self._create_results(some_branch, some_platform, some_builder, 'some-test', [50.0, 52.0], timestamps)
636         runs = Runs.update_or_insert(some_branch, some_platform, some_test)
637         self.assertOnlyInstance(runs)
638         self.assertEqual(json.loads('[' + runs.json_runs + ']'),
639             [[5, [4, 0, 100, None], mktime(timestamps[0].timetuple()), 50.0, 0, [], None, None],
640             [7, [6, 1, 101, None], mktime(timestamps[1].timetuple()), 52.0, 0, [], None, None]])
641         self.assertEqual(json.loads('{' + runs.json_averages + '}'), {"100": 50.0, "101": 52.0})
642         self.assertEqual(runs.json_min, 50.0)
643         self.assertEqual(runs.json_max, 52.0)
644
645         timestamps.append(datetime.now())
646         builds, results = self._create_results(some_branch, some_platform, some_builder, 'some-test', [48.0],
647             timestamps[2:], starting_revision=102)
648         runs.update_incrementally(builds[0], results[0])
649
650         self.assertOnlyInstance(runs)
651         self.assertEqual(json.loads('[' + runs.json_runs + ']'),
652             [[5, [4, 0, 100, None], mktime(timestamps[0].timetuple()), 50.0, 0, [], None, None],
653             [7, [6, 1, 101, None], mktime(timestamps[1].timetuple()), 52.0, 0, [], None, None],
654             [9, [8, 0, 102, None], mktime(timestamps[2].timetuple()), 48.0, 0, [], None, None]])
655         self.assertEqual(json.loads('{' + runs.json_averages + '}'), {"100": 50.0, "101": 52.0, "102": 48.0})
656         self.assertEqual(runs.json_min, 48.0)
657         self.assertEqual(runs.json_max, 52.0)
658
659     def test_json_by_ids(self):
660         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
661         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
662         some_builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
663         some_test = Test.update_or_insert('some-test', some_branch, some_platform)
664
665         self._create_results(some_branch, some_platform, some_builder, 'some-test', [50.0])
666         runs = Runs.update_or_insert(some_branch, some_platform, some_test)
667         runs_json = runs.to_json()
668
669         key_name = Runs._key_name(some_branch.id, some_platform.id, some_test.id)
670         self.assertEqual(Runs.json_by_ids(some_branch.id, some_platform.id, some_test.id), runs_json)
671         self.assertEqual(memcache.get(key_name), runs_json)
672
673         memcache.set(key_name, 'blah')
674         self.assertEqual(Runs.json_by_ids(some_branch.id, some_platform.id, some_test.id), 'blah')
675
676         memcache.delete(key_name)
677         self.assertEqual(Runs.json_by_ids(some_branch.id, some_platform.id, some_test.id), runs_json)
678         self.assertEqual(memcache.get(key_name), runs_json)
679
680     def test_to_json_without_results(self):
681         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
682         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
683         some_test = Test.update_or_insert('some-test', some_branch, some_platform)
684         self.assertOnlyInstance(some_test)
685         self.assertThereIsNoInstanceOf(TestResult)
686         self.assertEqual(json.loads(Runs.update_or_insert(some_branch, some_platform, some_test).to_json()), {
687             'test_runs': [],
688             'averages': {},
689             'min': None,
690             'max': None,
691             'unit': None,
692             'date_range': None,
693             'stat': 'ok'})
694
695     def test_to_json_with_results(self):
696         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
697         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
698         some_builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
699         some_test = Test.update_or_insert('some-test', some_branch, some_platform)
700         builds, results = self._create_results(some_branch, some_platform, some_builder, 'some-test', [50.0, 51.0, 52.0, 49.0, 48.0])
701
702         value = json.loads(Runs.update_or_insert(some_branch, some_platform, some_test).to_json())
703         self.assertEqualUnorderedList(value.keys(), ['test_runs', 'averages', 'min', 'max', 'unit', 'date_range', 'stat'])
704         self.assertEqual(value['stat'], 'ok')
705         self.assertEqual(value['min'], 48.0)
706         self.assertEqual(value['max'], 52.0)
707         self.assertEqual(value['unit'], None)
708         self.assertEqual(value['date_range'], None)  # date_range is never given
709
710         self.assertEqual(len(value['test_runs']), len(results))
711         for i, run in enumerate(value['test_runs']):
712             result = results[i]
713             self.assertEqual(run[0], result.key().id())
714             self.assertEqual(run[1][1], i)  # Build number
715             self.assertEqual(run[1][2], 100 + i)  # Revision
716             self.assertEqual(run[1][3], None)  # Supplementary revision
717             self.assertEqual(run[3], result.value)
718             self.assertEqual(run[6], some_builder.key().id())
719             self.assertEqual(run[7], None)  # Statistics
720
721     def test_to_json_with_unit(self):
722         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
723         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
724         some_builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
725         some_test = Test.update_or_insert('some-test', some_branch, some_platform, 'runs/s')
726         builds, results = self._create_results(some_branch, some_platform, some_builder, 'some-test', [50.0, 51.0, 52.0, 49.0, 48.0])
727
728         value = json.loads(Runs.update_or_insert(some_branch, some_platform, some_test).to_json())
729         self.assertEqual(value['unit'], 'runs/s')
730
731     def _assert_entry(self, entry, build, result, value, statistics=None, supplementary_revisions=None):
732         entry = entry[:]
733         entry[2] = None  # timestamp
734         self.assertEqual(entry, [result.key().id(), [build.key().id(), build.buildNumber, build.revision, supplementary_revisions],
735             None,  # timestamp
736             value, 0,  # runNumber
737             [],  # annotations
738             build.builder.key().id(), statistics])
739
740     def test_run_from_build_and_result(self):
741         branch = Branch.create_if_possible('some-branch', 'Some Branch')
742         platform = Platform.create_if_possible('some-platform', 'Some Platform')
743         builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
744         test_name = ' some-test'
745
746         def create_build(build_number, revision):
747             timestamp = datetime.now().replace(microsecond=0)
748             build = Build(branch=branch, platform=platform, builder=builder, buildNumber=build_number,
749                 revision=revision, timestamp=timestamp)
750             build.put()
751             return build
752
753         build = create_build(1, 101)
754         result = TestResult(name=test_name, value=123.0, build=build)
755         result.put()
756         self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 123.0)
757
758         build = create_build(2, 102)
759         result = TestResult(name=test_name, value=456.0, valueMedian=789.0, build=build)
760         result.put()
761         self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0)
762
763         result.valueStdev = 7.0
764         result.put()
765         self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0)
766
767         result.valueStdev = None
768         result.valueMin = 123.0
769         result.valueMax = 789.0
770         result.put()
771         self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0)
772
773         result.valueStdev = 8.0
774         result.valueMin = 123.0
775         result.valueMax = 789.0
776         result.put()
777         self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0,
778             statistics={'stdev': 8.0, 'min': 123.0, 'max': 789.0})
779
780         result.valueMedian = 345.0  # Median is never used by the frontend.
781         result.valueStdev = 8.0
782         result.valueMin = 123.0
783         result.valueMax = 789.0
784         result.put()
785         self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0,
786             statistics={'stdev': 8.0, 'min': 123.0, 'max': 789.0})
787
788     def test_chart_params_with_value(self):
789         some_branch = Branch.create_if_possible('some-branch', 'Some Branch')
790         some_platform = Platform.create_if_possible('some-platform', 'Some Platform')
791         some_builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
792         some_test = Test.update_or_insert('some-test', some_branch, some_platform)
793
794         start_time = datetime(2011, 2, 21, 12, 0, 0)
795         end_time = datetime(2011, 2, 28, 12, 0, 0)
796         results = self._create_results(some_branch, some_platform, some_builder, 'some-test',
797             [50.0, 51.0, 52.0, 49.0, 48.0, 51.9, 50.7, 51.1],
798             [start_time + timedelta(day) for day in range(0, 8)])
799
800         # Use int despite of its impreciseness since tests may fail due to rounding errors otherwise.
801         def split_as_int(string):
802             return [int(float(value)) for value in string.split(',')]
803
804         params = Runs.update_or_insert(some_branch, some_platform, some_test).chart_params(7)
805         self.assertEqual(params['chxl'], '0:|Feb 21|Feb 22|Feb 23|Feb 24|Feb 25|Feb 26|Feb 27|Feb 28')
806         self.assertEqual(split_as_int(params['chxr']), [1, 0, 57, int(52 * 1.1 / 5 + 0.5)])
807         x_min, x_max, y_min, y_max = split_as_int(params['chds'])
808         self.assertEqual(datetime.fromtimestamp(x_min), start_time)
809         self.assertEqual(datetime.fromtimestamp(x_max), end_time)
810         self.assertEqual(y_min, 0)
811         self.assertEqual(y_max, int(52 * 1.1))
812         self.assertEqual(split_as_int(params['chg']), [int(100 / 7), 20, 0, 0])
813
814         params = Runs.update_or_insert(some_branch, some_platform, some_test).chart_params(14)
815         self.assertEqual(params['chxl'], '0:|Feb 14|Feb 16|Feb 18|Feb 20|Feb 22|Feb 24|Feb 26|Feb 28')
816         self.assertEqual(split_as_int(params['chxr']), [1, 0, 57, int(52 * 1.1 / 5 + 0.5)])
817         x_min, x_max, y_min, y_max = split_as_int(params['chds'])
818         self.assertEqual(datetime.fromtimestamp(x_min), datetime(2011, 2, 14, 12, 0, 0))
819         self.assertEqual(datetime.fromtimestamp(x_max), end_time)
820         self.assertEqual(y_min, 0)
821         self.assertEqual(y_max, int(52 * 1.1))
822         self.assertEqual(split_as_int(params['chg']), [int(100 / 7), 20, 0, 0])
823
824
825 class DashboardImageTests(DataStoreTestsBase):
826     def setUp(self):
827         super(DashboardImageTests, self).setUp()
828         self.testbed.init_memcache_stub()
829
830     def test_create(self):
831         self.assertEqual(memcache.get('dashboard-image:1:2:3:7'), None)
832         self.assertThereIsNoInstanceOf(DashboardImage)
833         image = DashboardImage.create(1, 2, 3, 7, 'blah')
834         self.assertOnlyInstance(image)
835         self.assertEqual(memcache.get('dashboard-image:1:2:3:7'), 'blah')
836
837     def test_get(self):
838         image = DashboardImage.create(1, 2, 3, 7, 'blah')
839         self.assertEqual(memcache.get('dashboard-image:1:2:3:7'), 'blah')
840         memcache.set('dashboard-image:1:2:3:7', 'new value')
841
842         # Check twice to make sure the first call doesn't clear memcache
843         self.assertEqual(DashboardImage.get_image(1, 2, 3, 7), 'new value')
844         self.assertEqual(DashboardImage.get_image(1, 2, 3, 7), 'new value')
845
846         memcache.delete('dashboard-image:1:2:3:7')
847         self.assertEqual(memcache.get('dashboard-image:1:2:3:7'), None)
848         self.assertEqual(DashboardImage.get_image(1, 2, 3, 7), 'blah')
849         self.assertEqual(memcache.get('dashboard-image:1:2:3:7'), 'blah')
850
851     def test_needs_update(self):
852         self.assertTrue(DashboardImage.needs_update(1, 2, 3, 7))
853         self.assertTrue(DashboardImage.needs_update(1, 2, 3, 30))
854         self.assertTrue(DashboardImage.needs_update(1, 2, 3, 60))
855         self.assertTrue(DashboardImage.needs_update(1, 2, 3, 365))
856
857         image = DashboardImage(key_name=DashboardImage.key_name(1, 2, 3, 7), image='blah')
858         image.put()
859         self.assertOnlyInstance(image)
860         self.assertTrue(DashboardImage.needs_update(1, 2, 3, 7))
861
862         DashboardImage(key_name=DashboardImage.key_name(1, 2, 3, 30), image='blah').put()
863         self.assertFalse(DashboardImage.needs_update(1, 2, 3, 30, datetime.now() + timedelta(0, 10)))
864
865         DashboardImage(key_name=DashboardImage.key_name(1, 2, 4, 30), image='blah').put()
866         self.assertTrue(DashboardImage.needs_update(1, 2, 4, 30, datetime.now() + timedelta(1)))
867
868         DashboardImage(key_name=DashboardImage.key_name(1, 2, 3, 90), image='blah').put()
869         self.assertFalse(DashboardImage.needs_update(1, 2, 3, 90, datetime.now() + timedelta(0, 20)))
870
871         DashboardImage(key_name=DashboardImage.key_name(1, 2, 4, 90), image='blah').put()
872         self.assertTrue(DashboardImage.needs_update(1, 2, 4, 90, datetime.now() + timedelta(1)))
873
874         DashboardImage(key_name=DashboardImage.key_name(1, 2, 3, 365), image='blah').put()
875         self.assertFalse(DashboardImage.needs_update(1, 2, 3, 365, datetime.now() + timedelta(1)))
876
877         DashboardImage(key_name=DashboardImage.key_name(1, 2, 4, 365), image='blah').put()
878         self.assertTrue(DashboardImage.needs_update(1, 2, 4, 365, datetime.now() + timedelta(10)))
879
880
881 if __name__ == '__main__':
882     unittest.main()