Rewrite generate-xcfilelists in Python
[WebKit-https.git] / Tools / Scripts / webkitpy / generate_xcfilelists_lib / application.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3 #
4 # Copyright (C) 2019 Apple Inc.  All rights reserved.
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions
8 # are met:
9 #
10 # 1. Redistributions of source code must retain the above copyright
11 #    notice, this list of conditions and the following disclaimer.
12 # 2. Redistributions in binary form must reproduce the above copyright
13 #    notice, this list of conditions and the following disclaimer in the
14 #    documentation and/or other materials provided with the distribution.
15 #
16 # THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
17 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23 # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24 # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28 # Application represents the main operation of the script. It's a singleton,
29 # created by main() and then invoked to run everything. This class parses the
30 # command-line options, validates them, creates and invokes BaseGenerators,
31 # reports the results, and handles the catching and reporting of any
32 # exceptions/errors.
33
34 from __future__ import print_function
35
36 import argparse
37 import itertools
38 import os
39 import sys
40 import textwrap
41 import traceback
42
43 from functools import reduce
44
45 import webkitpy.generate_xcfilelists_lib.generators as Generators
46 import webkitpy.generate_xcfilelists_lib.util as util
47
48
49 EX_GENERAL_ERROR = 1  # General error
50 EX_ACTION_REQUIRED = 2  # Returned when script determines that the generated .xcfilelist files have changed.
51
52
53 class Application(object):
54
55     __slots__ = (
56         "command_file",
57         "parser", "cmd_line_args",
58         "dispatch", "project_specific_generators",
59         "supported_project_tags", "supported_platforms", "supported_configurations")
60
61     # Aliases for platforms. These are handy when using the script from the
62     # command line and you don't remember if it's "ios" or "iphoneos, or "tvos"
63     # or "appletvos". This list of aliases will let you use any of those.
64
65     platform_aliases = {
66         "ios":          "iphoneos",
67         "iphone":       "iphoneos",
68         "simulator":    "iphonesimulator",
69         "sim":          "iphonesimulator",
70         "mac":          "macosx",
71         "macos":        "macosx",
72         "osx":          "macosx",
73         "tvos":         "appletvos",
74         "tv":           "appletvos",
75         "tvsimulator":  "appletvsimulator",
76         "watch":        "watchos",
77     }
78
79     @util.LogEntryExit
80     def __init__(self, command_file):
81         self.command_file = os.path.realpath(command_file)
82         self.parser = None
83         self.cmd_line_args = None
84
85         self.dispatch = {
86             "generate":       self._cmd_set_environment_and_generate,
87             "generate-xcode": self._cmd_generate_within_xcode,
88             "check":          self._cmd_set_environment_and_check,
89             "check-xcode":    self._cmd_check_within_xcode,
90             "generate-inner": self._cmd_generate_within_xcode_and_return_results_to_caller,
91             "help":           self._cmd_help,
92         }
93
94         self.project_specific_generators = {
95             "JavaScriptCore":   Generators.JavaScriptCoreGenerator,
96             "WebCore":          Generators.WebCoreGenerator,
97             "WebKit":           Generators.WebKitGenerator,
98             "DumpRenderTree":   Generators.DumpRenderTreeGenerator,
99             "WebKitTestRunner": Generators.WebKitTestRunnerGenerator,
100         }
101
102         self.supported_project_tags = None
103         self.supported_platforms = None
104         self.supported_configurations = None
105
106     @util.LogEntryExit
107     def run(self):
108         try:
109             self._initialize()
110
111             self.parser = self._create_parser()
112             self.cmd_line_args = args = self.parser.parse_args()
113
114             if args.help:
115                 return self._cmd_help(os.EX_OK)
116
117             self._validate_args(args)
118
119             try:
120                 func = self.dispatch[args.command]
121             except KeyError as e:
122                 raise util.InvalidCommandError(args.command)
123
124             return func()
125
126         except util.InvalidArgumentError as e:
127             print("### Invalid argument: {}".format(e))
128             return self._cmd_help(os.EX_USAGE)
129
130         except util.InvalidCommandError as e:
131             if e.args:
132                 print("### Invalid command: {}".format(e))
133             else:
134                 print("### Missing command")
135             return self._cmd_help(os.EX_USAGE)
136
137         except SystemExit:
138             raise
139
140         except BaseException as e:
141             traceback.print_exc()
142             return os.EX_SOFTWARE
143
144     # Perform some post __init__ initialization. This is performed after
145     # __init__ so that we can respond to any information provided in any
146     # sub-class's __init__.
147
148     @util.LogEntryExit
149     def _initialize(self):
150         def collect_attributes(key):
151             configurations = set()
152             for project_tag in self.project_specific_generators:
153                 configurations |= set(key(self.project_specific_generators[project_tag]))
154             return configurations
155
156         self.supported_project_tags = sorted(list(self.project_specific_generators.keys()))
157         self.supported_platforms = sorted(list(collect_attributes(lambda gen_cls: gen_cls.VALID_PLATFORMS)))
158         self.supported_configurations = sorted(list(collect_attributes(lambda gen_cls: gen_cls.VALID_CONFIGURATIONS)))
159
160     @util.LogEntryExit
161     def _create_parser(self):
162         valid_commands = ("generate", "generate-xcode", "check", "check-xcode", "generate-inner", "help")
163         valid_commands_prompt = "(" + " | ".join(valid_commands) + ")"
164
165         parser = argparse.ArgumentParser(add_help=False,
166                 formatter_class=argparse.RawDescriptionHelpFormatter,
167                 description="""\
168 Generate or check .xcfilelist files. One of the following commands must be
169 specified on the command-line:
170
171   generate              Generate a complete and up-to-date set of .xcfilelist
172                         files and copy them to their appropriate places in the
173                         project directories.
174   generate-xcode        Similar to generate, but to be called from within Xcode.
175   check                 Generate a complete and up-to-date set of .xcfilelist
176                         files and compare them to their counterparts in the
177                         project directories.
178   check-xcode           Similar to check, but to be called from within Xcode.
179   generate-inner        [Used by script internals] Generate an .xcfilelist file
180                         for a particular combination of project, platform, and
181                         configuration. This operation is performed in the
182                         context of an Xcode build in order to inherit the same
183                         environment as that build. Once generated, the results
184                         are returned to the calling instance of this script.
185   help                  Print this text and exit.""")
186
187         parser.add_argument("command", action=util.CheckCommandAction,
188                 valid_commands=valid_commands, metavar=valid_commands_prompt,
189                 help="""\
190                         The operation to perform.""")
191
192         parser.add_argument("--project", action=util.CheckValidItemAction,
193                 item_type="project",
194                 valid_items=self.supported_project_tags,
195                 dest="project_tags", metavar="<PROJECT>", help="""\
196                         Specify which project or projects for which to generate
197                         .xcfilelist files or to check. Possible values are
198                         ({}). Can be specified more than once. Default is to
199                         iterate over all projects.""".format(
200                             ", ".join(self.supported_project_tags)))
201         parser.add_argument("--platform", action=util.CheckValidItemAction,
202                 item_type="platform",
203                 valid_items=self.supported_platforms,
204                 aliases=self.platform_aliases,
205                 dest="platforms", metavar="<PLATFORM>", help="""\
206                         Specify which platform or platforms for which to
207                         generate .xcfilelist files or to check. Possible values
208                         are ({}, plus common aliases). Can be specified more
209                         than once. Default is to iterate over all platforms,
210                         filtered to those platforms that a particular project
211                         supports (e.g., you can't specify 'iphoneos' for
212                         WebKitTestRunner).""".format(
213                             ", ".join(self.supported_platforms)))
214         parser.add_argument("--configuration", action=util.CheckValidItemAction,
215                 item_type="configuration",
216                 valid_items=self.supported_configurations,
217                 dest="configurations", metavar="<CONFIGURATION>", help="""\
218                         Specify which configuration or configurations for which
219                         to generate .xcfilelist files or to check. Possible
220                         values are ({}). Can be specified more than once.
221                         Default is to iterate over all
222                         configurations.""".format(
223                             ", ".join(self.supported_configurations)))
224         parser.add_argument("--xcode", metavar="<WORKSPACE>", help="""\
225                         If the existing build output was created by building
226                         with Xcode, specify the path to the workspace that was
227                         used.""")
228         parser.add_argument("-d", "--debug", action="store_true", help="""\
229                         Provide verbose output.""")
230         parser.add_argument("--debug-file", help="""\
231                         [Used by script internals] Name of the file to which to
232                         write debug information. Used when this script
233                         sub-launches itself and needs to collect the debug
234                         information from the sub-launched instance. Not
235                         normally used when this script is invoked from the
236                         command-line or Xcode.""")
237         parser.add_argument("--pickle-file", help="""\
238                         [Used by script internals] Name of the file used to
239                         store results to be transported out from the Xcode
240                         execution environment out to an outer layer. This
241                         parameter is only used with the 'generate-core'
242                         command.""")
243         parser.add_argument("-q", "--quiet", action="store_true", help="""\
244                         Don't print any standard output.""")
245         parser.add_argument("-h", "--help", action="store_true", help="""\
246                         Print this text and exit.""")
247
248         setattr(parser, "application", self)
249         return parser
250
251     @util.LogEntryExit
252     def _validate_args(self, args):
253         if not self.cmd_line_args.project_tags:
254             self.cmd_line_args.project_tags = self.supported_project_tags
255         if not self.cmd_line_args.platforms:
256             self.cmd_line_args.platforms = self.supported_platforms
257         if not self.cmd_line_args.configurations:
258             self.cmd_line_args.configurations = self.supported_configurations
259
260         if util.is_running_under_xcode():
261             assert len(self.cmd_line_args.project_tags) == 1
262             assert len(self.cmd_line_args.platforms) == 1
263             assert len(self.cmd_line_args.configurations) == 1
264
265     @util.LogEntryExit
266     def _cmd_set_environment_and_generate(self):
267         generators = self._do_set_environment_and_generate()
268         generators = self._do_merge(generators)
269         return self._report_results(generators)
270
271     @util.LogEntryExit
272     def _cmd_generate_within_xcode(self):
273         generators = self._do_generate()
274         generators = self._do_merge(generators)
275         return self._report_results(generators)
276
277     @util.LogEntryExit
278     def _cmd_set_environment_and_check(self):
279         generators = self._do_set_environment_and_generate()
280         return self._report_results(generators)
281
282     @util.LogEntryExit
283     def _cmd_check_within_xcode(self):
284         generators = self._do_generate()
285         return self._report_results(generators)
286
287     @util.LogEntryExit
288     def _cmd_generate_within_xcode_and_return_results_to_caller(self):
289         generators = self._do_generate()
290         with open(self.cmd_line_args.pickle_file, "wb") as f:
291             for generator in generators:
292                 generator.pickle_to_file(f)
293         return os.EX_OK
294
295     @util.LogEntryExit
296     def _cmd_help(self, status=os.EX_OK):
297         self.parser.print_help()
298         return status
299
300     @util.LogEntryExit
301     def _do_set_environment_and_generate(self):
302         def core_operation(generator, generators):
303             new_generators = generator.set_environment_and_generate()
304             generators.extend(new_generators)
305             return generators
306         return self._do_generate_common(core_operation)
307
308     @util.LogEntryExit
309     def _do_generate(self):
310         def core_operation(generator, generators):
311             generator.generate()
312             generators.append(generator)
313             return generators
314         return self._do_generate_common(core_operation)
315
316     @util.LogEntryExit
317     def _do_generate_common(self, core_operation):
318         generators = []
319
320         for triple in itertools.product(
321                 self.cmd_line_args.project_tags,
322                 self.cmd_line_args.platforms,
323                 self.cmd_line_args.configurations):
324             generator = self.project_specific_generators[triple[0]](self, *triple)
325             if not generator.is_valid():
326                 continue
327             self._log_progress("Generating .xcfilelists for {}/{}/{}".format(*triple))
328             try:
329                 generators = core_operation(generator, generators)
330             except BaseException as e:
331                 # TODO: Turn the traceback into a string, and then allow
332                 # this field to be pickled and printed by the calling
333                 # context. Right now, pickling raises an exception if it
334                 # encounters a Traceback object. See BaseGenerator.pickle_to_file.
335                 (generator.ex_type, generator.ex_value, generator.ex_traceback) = sys.exc_info()
336             if generator.has_error():
337                 sys.exit(self._report_results([generator]))
338
339         return generators
340
341     @util.LogEntryExit
342     def _do_merge(self, generators):
343         if self._any_have_errors(generators):
344             return generators
345
346         for generator in generators:
347             if generator.has_action():
348                 self._log_progress("Merging .xcfilelists for {}/{}/{}".format(*generator.triple))
349                 generator.merge()
350
351         return generators
352
353     @util.LogEntryExit
354     def _report_results(self, generators):
355         generators_with_errors = [generator for generator in generators if generator.has_error()]
356         if generators_with_errors:
357             for generator in generators_with_errors:
358                 generator.report_error()
359             return EX_GENERAL_ERROR
360
361         generators_with_actions = [generator for generator in generators if generator.has_action()]
362         if generators_with_actions:
363             if self.cmd_line_args.command == "generate" or self.cmd_line_args.command == "generate-xcode":
364                 self._report_merge_results(generators_with_actions)
365             else:
366                 self._report_remediation_steps(generators_with_actions)
367             return EX_ACTION_REQUIRED
368
369         return os.EX_OK
370
371     @util.LogEntryExit
372     def _report_merge_results(self, generators):
373         message = textwrap.wrap(
374                 "\".xcfilelist\" files tell the build system what files are " +
375                 "consumed and produced by the \"Run Script\" build phases in " +
376                 "Xcode. At least one of these .xcfilelist files was out of date " +
377                 "and has been updated. You now need to restart your build.", 90)
378
379         self._log_results("")
380         for line in message:
381             self._log_results(line)
382
383     @util.LogEntryExit
384     def _report_remediation_steps(self, generators):
385         message = textwrap.wrap("One or more \".xcfilelist\" files are out of date. Regenerate them by running the following commands:", 90)
386         message.append("")
387
388         def add_to_message(generator, message):
389             if generator.has_action():
390                 message.append("    `Tools/Scripts/generate-xcfilelists generate --project {} --platform {} --configuration {}{}`\n".format(
391                         generator.project_tag, generator.platform, generator.configuration,
392                         " --xcode {}".format(self.cmd_line_args.xcode) if self.cmd_line_args.xcode else ""))
393             return message
394
395         for generator in generators:
396             message = add_to_message(generator, message)
397
398         for line in message:
399             self._log_results(line)
400
401     @util.LogEntryExit
402     def _any_have_errors(self, generators):
403         return reduce(lambda acc, generator: acc or generator.has_error(), generators, None)
404
405     @util.LogEntryExit
406     def _any_have_actions(self, generators):
407         return reduce(lambda acc, generator: acc or generator.has_action(), generators, None)
408
409     @util.LogEntryExit
410     def _log_progress(self, message):
411         if not self.cmd_line_args.quiet:
412             print("### {}".format(message))
413
414     @util.LogEntryExit
415     def _log_results(self, message):
416         if not self.cmd_line_args.quiet:
417             print("{}".format(message))
418
419     # Return the path to the script to sublaunch.
420
421     @util.LogEntryExit
422     def get_generate_xcfilelists_script_path(self):
423         return self.command_file
424
425     # Return the parent of the WebKit check-out directory.
426
427     @util.LogEntryExit
428     def _get_root_parent_dir(self):
429         return os.path.dirname(     # Remove "OpenSource"
430                 os.path.dirname(        # Remove "Tools"
431                     os.path.dirname(        # Remove "Scripts"
432                         os.path.dirname(        # Remove script name
433                             self.get_generate_xcfilelists_script_path()))))
434
435     # Return the path to the WebKit check-out directory.
436
437     @util.LogEntryExit
438     def get_opensource_dir(self):
439         return os.path.join(self._get_root_parent_dir(), "OpenSource")
440
441     # Return the path to the directory containing supporting build scripts.
442
443     @util.LogEntryExit
444     def get_build_scripts_dir(self):
445         return os.path.join(self.get_opensource_dir(), "Source", "WTF", "Scripts")
446
447     # Return the path to a supporting build script.
448
449     @util.LogEntryExit
450     def get_extract_dependencies_from_makefile_script(self):
451         return os.path.join(self.get_opensource_dir(), "Tools", "Scripts", "extract-dependencies-from-makefile")
452
453     # Return $(BUILT_PRODUCTS_DIR)
454     # aka $(CONFIGURATION_BUILD_DIR)
455     # aka $(BUILD_DIR)/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)
456
457     @util.LogEntryExit
458     def get_xcode_built_products_dir(self):
459         assert util.is_running_under_xcode()
460         return self._getenv("BUILT_PRODUCTS_DIR")
461
462     # Return the named environment variable.
463     @util.LogEntryExit
464     def _getenv(self, variable_name):
465         return os.environ.get(variable_name)