Add a verbose flag to binding tests
[WebKit-https.git] / Tools / Scripts / webkitpy / bindings / main.py
1 # Copyright (C) 2011 Google Inc.  All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1. Redistributions of source code must retain the above copyright
7 #    notice, this list of conditions and the following disclaimer.
8 # 2. Redistributions in binary form must reproduce the above copyright
9 #    notice, this list of conditions and the following disclaimer in the
10 #    documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
13 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
14 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
16 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
17 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
18 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
19 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
20 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 #
24
25 import os
26 import os.path
27 import shutil
28 import subprocess
29 import sys
30 import tempfile
31 from webkitpy.common.checkout.scm.detection import detect_scm_system
32 from webkitpy.common.system.executive import ScriptError
33
34
35 class BindingsTests:
36
37     def __init__(self, reset_results, generators, executive, verbose):
38         self.reset_results = reset_results
39         self.generators = generators
40         self.executive = executive
41         self.verbose = verbose
42
43     def generate_from_idl(self, generator, idl_file, output_directory, supplemental_dependency_file):
44         cmd = ['perl', '-w',
45                '-IWebCore/bindings/scripts',
46                'WebCore/bindings/scripts/generate-bindings.pl',
47                # idl include directories (path relative to generate-bindings.pl)
48                '--include', '.',
49                '--defines', 'TESTING_%s' % generator,
50                '--generator', generator,
51                '--outputDir', output_directory,
52                '--supplementalDependencyFile', supplemental_dependency_file,
53                idl_file]
54
55         exit_code = 0
56         try:
57             output = self.executive.run_command(cmd)
58             if output:
59                 print output
60         except ScriptError, e:
61             print e.output
62             exit_code = e.exit_code
63         return exit_code
64
65     def generate_supplemental_dependency(self, input_directory, supplemental_dependency_file, window_constructors_file, workerglobalscope_constructors_file, sharedworkerglobalscope_constructors_file, dedicatedworkerglobalscope_constructors_file):
66         idl_files_list = tempfile.mkstemp()
67         for input_file in os.listdir(input_directory):
68             (name, extension) = os.path.splitext(input_file)
69             if extension != '.idl':
70                 continue
71             os.write(idl_files_list[0], os.path.join(input_directory, input_file) + "\n")
72         os.close(idl_files_list[0])
73
74         cmd = ['perl', '-w',
75                '-IWebCore/bindings/scripts',
76                'WebCore/bindings/scripts/preprocess-idls.pl',
77                '--idlFilesList', idl_files_list[1],
78                '--defines', '',
79                '--supplementalDependencyFile', supplemental_dependency_file,
80                '--windowConstructorsFile', window_constructors_file,
81                '--workerGlobalScopeConstructorsFile', workerglobalscope_constructors_file,
82                '--sharedWorkerGlobalScopeConstructorsFile', sharedworkerglobalscope_constructors_file,
83                '--dedicatedWorkerGlobalScopeConstructorsFile', dedicatedworkerglobalscope_constructors_file]
84
85         exit_code = 0
86         try:
87             output = self.executive.run_command(cmd)
88             if output:
89                 print output
90         except ScriptError, e:
91             print e.output
92             exit_code = e.exit_code
93         os.remove(idl_files_list[1])
94         return exit_code
95
96     def detect_changes(self, generator, work_directory, reference_directory):
97         changes_found = False
98         for output_file in os.listdir(work_directory):
99             cmd = ['diff',
100                    '-u',
101                    '-N',
102                    os.path.join(reference_directory, output_file),
103                    os.path.join(work_directory, output_file)]
104
105             exit_code = 0
106             try:
107                 output = self.executive.run_command(cmd)
108             except ScriptError, e:
109                 output = e.output
110                 exit_code = e.exit_code
111
112             if exit_code or output:
113                 print 'FAIL: (%s) %s' % (generator, output_file)
114                 print output
115                 changes_found = True
116             elif self.verbose:
117                 print 'PASS: (%s) %s' % (generator, output_file)
118         return changes_found
119
120     def run_tests(self, generator, input_directory, reference_directory, supplemental_dependency_file):
121         work_directory = reference_directory
122
123         passed = True
124         for input_file in os.listdir(input_directory):
125             (name, extension) = os.path.splitext(input_file)
126             if extension != '.idl':
127                 continue
128             # Generate output into the work directory (either the given one or a
129             # temp one if not reset_results is performed)
130             if not self.reset_results:
131                 work_directory = tempfile.mkdtemp()
132
133             if self.generate_from_idl(generator,
134                                       os.path.join(input_directory, input_file),
135                                       work_directory,
136                                       supplemental_dependency_file):
137                 passed = False
138
139             if self.reset_results:
140                 print "Reset results: (%s) %s" % (generator, input_file)
141                 continue
142
143             # Detect changes
144             if self.detect_changes(generator, work_directory, reference_directory):
145                 passed = False
146             shutil.rmtree(work_directory)
147
148         return passed
149
150     def main(self):
151         current_scm = detect_scm_system(os.curdir)
152         os.chdir(os.path.join(current_scm.checkout_root, 'Source'))
153
154         all_tests_passed = True
155
156         input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test')
157         supplemental_dependency_file = tempfile.mkstemp()[1]
158         window_constructors_file = tempfile.mkstemp()[1]
159         workerglobalscope_constructors_file = tempfile.mkstemp()[1]
160         sharedworkerglobalscope_constructors_file = tempfile.mkstemp()[1]
161         dedicatedworkerglobalscope_constructors_file = tempfile.mkstemp()[1]
162         if self.generate_supplemental_dependency(input_directory, supplemental_dependency_file, window_constructors_file, workerglobalscope_constructors_file, sharedworkerglobalscope_constructors_file, dedicatedworkerglobalscope_constructors_file):
163             print 'Failed to generate a supplemental dependency file.'
164             os.remove(supplemental_dependency_file)
165             os.remove(window_constructors_file)
166             os.remove(workerglobalscope_constructors_file)
167             os.remove(sharedworkerglobalscope_constructors_file)
168             os.remove(dedicatedworkerglobalscope_constructors_file)
169             return -1
170
171         for generator in self.generators:
172             input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test')
173             reference_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test', generator)
174             if not self.run_tests(generator, input_directory, reference_directory, supplemental_dependency_file):
175                 all_tests_passed = False
176
177         os.remove(supplemental_dependency_file)
178         os.remove(window_constructors_file)
179         os.remove(workerglobalscope_constructors_file)
180         os.remove(sharedworkerglobalscope_constructors_file)
181         os.remove(dedicatedworkerglobalscope_constructors_file)
182         print ''
183         if all_tests_passed:
184             print 'All tests PASS!'
185             return 0
186         else:
187             print 'Some tests FAIL! (To update the reference files, execute "run-bindings-tests --reset-results")'
188             return -1