Add the ability to filter the set of bindings tests you want to run
[WebKit-https.git] / Tools / Scripts / webkitpy / bindings / main.py
1 # Copyright (C) 2011 Google Inc.  All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions
5 # are met:
6 # 1. Redistributions of source code must retain the above copyright
7 #    notice, this list of conditions and the following disclaimer.
8 # 2. Redistributions in binary form must reproduce the above copyright
9 #    notice, this list of conditions and the following disclaimer in the
10 #    documentation and/or other materials provided with the distribution.
11 #
12 # THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
13 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
14 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
15 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
16 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
17 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
18 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
19 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
20 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
21 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
22 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23 #
24
25 import fnmatch
26 import os
27 import os.path
28 import shutil
29 import subprocess
30 import sys
31 import tempfile
32 from webkitpy.common.checkout.scm.detection import detect_scm_system
33 from webkitpy.common.system.executive import ScriptError
34
35
36 class BindingsTests:
37
38     def __init__(self, reset_results, generators, executive, verbose, patterns):
39         self.reset_results = reset_results
40         self.generators = generators
41         self.executive = executive
42         self.verbose = verbose
43         self.patterns = patterns
44
45     def generate_from_idl(self, generator, idl_file, output_directory, supplemental_dependency_file):
46         cmd = ['perl', '-w',
47                '-IWebCore/bindings/scripts',
48                'WebCore/bindings/scripts/generate-bindings.pl',
49                # idl include directories (path relative to generate-bindings.pl)
50                '--include', '.',
51                '--defines', 'TESTING_%s' % generator,
52                '--generator', generator,
53                '--outputDir', output_directory,
54                '--supplementalDependencyFile', supplemental_dependency_file,
55                idl_file]
56
57         exit_code = 0
58         try:
59             output = self.executive.run_command(cmd)
60             if output:
61                 print output
62         except ScriptError, e:
63             print e.output
64             exit_code = e.exit_code
65         return exit_code
66
67     def generate_supplemental_dependency(self, input_directory, supplemental_dependency_file, window_constructors_file, workerglobalscope_constructors_file, dedicatedworkerglobalscope_constructors_file):
68         idl_files_list = tempfile.mkstemp()
69         for input_file in os.listdir(input_directory):
70             (name, extension) = os.path.splitext(input_file)
71             if extension != '.idl':
72                 continue
73             os.write(idl_files_list[0], os.path.join(input_directory, input_file) + "\n")
74         os.close(idl_files_list[0])
75
76         cmd = ['perl', '-w',
77                '-IWebCore/bindings/scripts',
78                'WebCore/bindings/scripts/preprocess-idls.pl',
79                '--idlFilesList', idl_files_list[1],
80                '--defines', '',
81                '--supplementalDependencyFile', supplemental_dependency_file,
82                '--windowConstructorsFile', window_constructors_file,
83                '--workerGlobalScopeConstructorsFile', workerglobalscope_constructors_file,
84                '--dedicatedWorkerGlobalScopeConstructorsFile', dedicatedworkerglobalscope_constructors_file]
85
86         exit_code = 0
87         try:
88             output = self.executive.run_command(cmd)
89             if output:
90                 print output
91         except ScriptError, e:
92             print e.output
93             exit_code = e.exit_code
94         os.remove(idl_files_list[1])
95         return exit_code
96
97     def detect_changes(self, generator, work_directory, reference_directory):
98         changes_found = False
99         for output_file in os.listdir(work_directory):
100             cmd = ['diff',
101                    '-u',
102                    '-N',
103                    os.path.join(reference_directory, output_file),
104                    os.path.join(work_directory, output_file)]
105
106             exit_code = 0
107             try:
108                 output = self.executive.run_command(cmd)
109             except ScriptError, e:
110                 output = e.output
111                 exit_code = e.exit_code
112
113             if exit_code or output:
114                 print 'FAIL: (%s) %s' % (generator, output_file)
115                 print output
116                 changes_found = True
117             elif self.verbose:
118                 print 'PASS: (%s) %s' % (generator, output_file)
119         return changes_found
120
121     def test_matches_patterns(self, test):
122         if not self.patterns:
123             return True
124         for pattern in self.patterns:
125             if fnmatch.fnmatch(test, pattern):
126                 return True
127         return False
128
129     def run_tests(self, generator, input_directory, reference_directory, supplemental_dependency_file):
130         work_directory = reference_directory
131
132         passed = True
133         for input_file in os.listdir(input_directory):
134             (name, extension) = os.path.splitext(input_file)
135             if extension != '.idl':
136                 continue
137
138             if not self.test_matches_patterns(input_file):
139                 continue
140
141             # Generate output into the work directory (either the given one or a
142             # temp one if not reset_results is performed)
143             if not self.reset_results:
144                 work_directory = tempfile.mkdtemp()
145
146             if self.generate_from_idl(generator,
147                                       os.path.join(input_directory, input_file),
148                                       work_directory,
149                                       supplemental_dependency_file):
150                 passed = False
151
152             if self.reset_results:
153                 print "Reset results: (%s) %s" % (generator, input_file)
154                 continue
155
156             # Detect changes
157             if self.detect_changes(generator, work_directory, reference_directory):
158                 passed = False
159             shutil.rmtree(work_directory)
160
161         return passed
162
163     def main(self):
164         current_scm = detect_scm_system(os.curdir)
165         os.chdir(os.path.join(current_scm.checkout_root, 'Source'))
166
167         all_tests_passed = True
168
169         input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test')
170         supplemental_dependency_file = tempfile.mkstemp()[1]
171         window_constructors_file = tempfile.mkstemp()[1]
172         workerglobalscope_constructors_file = tempfile.mkstemp()[1]
173         dedicatedworkerglobalscope_constructors_file = tempfile.mkstemp()[1]
174         if self.generate_supplemental_dependency(input_directory, supplemental_dependency_file, window_constructors_file, workerglobalscope_constructors_file, dedicatedworkerglobalscope_constructors_file):
175             print 'Failed to generate a supplemental dependency file.'
176             os.remove(supplemental_dependency_file)
177             os.remove(window_constructors_file)
178             os.remove(workerglobalscope_constructors_file)
179             os.remove(dedicatedworkerglobalscope_constructors_file)
180             return -1
181
182         for generator in self.generators:
183             input_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test')
184             reference_directory = os.path.join('WebCore', 'bindings', 'scripts', 'test', generator)
185             if not self.run_tests(generator, input_directory, reference_directory, supplemental_dependency_file):
186                 all_tests_passed = False
187
188         os.remove(supplemental_dependency_file)
189         os.remove(window_constructors_file)
190         os.remove(workerglobalscope_constructors_file)
191         os.remove(dedicatedworkerglobalscope_constructors_file)
192         print ''
193         if all_tests_passed:
194             print 'All tests PASS!'
195             return 0
196         else:
197             print 'Some tests FAIL! (To update the reference files, execute "run-bindings-tests --reset-results")'
198             return -1