Merge r169628 from ftlopt.
[WebKit-https.git] / Source / JavaScriptCore / ftl / FTLCompile.cpp
1 /*
2  * Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "FTLCompile.h"
28
29 #if ENABLE(FTL_JIT)
30
31 #include "CodeBlockWithJITType.h"
32 #include "CCallHelpers.h"
33 #include "DFGCommon.h"
34 #include "DFGGraphSafepoint.h"
35 #include "DataView.h"
36 #include "Disassembler.h"
37 #include "FTLExitThunkGenerator.h"
38 #include "FTLInlineCacheSize.h"
39 #include "FTLJITCode.h"
40 #include "FTLThunks.h"
41 #include "FTLUnwindInfo.h"
42 #include "JITStubs.h"
43 #include "LLVMAPI.h"
44 #include "LinkBuffer.h"
45 #include "RepatchBuffer.h"
46
47 namespace JSC { namespace FTL {
48
49 using namespace DFG;
50
51 static uint8_t* mmAllocateCodeSection(
52     void* opaqueState, uintptr_t size, unsigned alignment, unsigned, const char* sectionName)
53 {
54     State& state = *static_cast<State*>(opaqueState);
55     
56     RELEASE_ASSERT(alignment <= jitAllocationGranule);
57     
58     RefPtr<ExecutableMemoryHandle> result =
59         state.graph.m_vm.executableAllocator.allocate(
60             state.graph.m_vm, size, state.graph.m_codeBlock, JITCompilationMustSucceed);
61     
62     // LLVM used to put __compact_unwind in a code section. We keep this here defensively,
63     // for clients that use older LLVMs.
64     if (!strcmp(sectionName, "__compact_unwind")) {
65         state.compactUnwind = result->start();
66         state.compactUnwindSize = result->sizeInBytes();
67     }
68     
69     state.jitCode->addHandle(result);
70     state.codeSectionNames.append(sectionName);
71     
72     return static_cast<uint8_t*>(result->start());
73 }
74
75 static uint8_t* mmAllocateDataSection(
76     void* opaqueState, uintptr_t size, unsigned alignment, unsigned sectionID,
77     const char* sectionName, LLVMBool isReadOnly)
78 {
79     UNUSED_PARAM(sectionID);
80     UNUSED_PARAM(isReadOnly);
81
82     // Allocate the GOT in the code section to make it reachable for all code.
83     if (!strcmp(sectionName, "__got"))
84         return mmAllocateCodeSection(opaqueState, size, alignment, sectionID, sectionName);
85
86     State& state = *static_cast<State*>(opaqueState);
87
88     RefPtr<DataSection> section = adoptRef(new DataSection(size, alignment));
89
90     if (!strcmp(sectionName, "__llvm_stackmaps"))
91         state.stackmapsSection = section;
92     else {
93         state.jitCode->addDataSection(section);
94         state.dataSectionNames.append(sectionName);
95         if (!strcmp(sectionName, "__compact_unwind")) {
96             state.compactUnwind = section->base();
97             state.compactUnwindSize = size;
98         }
99     }
100
101     return bitwise_cast<uint8_t*>(section->base());
102 }
103
104 static LLVMBool mmApplyPermissions(void*, char**)
105 {
106     return false;
107 }
108
109 static void mmDestroy(void*)
110 {
111 }
112
113 static void dumpDataSection(DataSection* section, const char* prefix)
114 {
115     for (unsigned j = 0; j < section->size() / sizeof(int64_t); ++j) {
116         char buf[32];
117         int64_t* wordPointer = static_cast<int64_t*>(section->base()) + j;
118         snprintf(buf, sizeof(buf), "0x%lx", static_cast<unsigned long>(bitwise_cast<uintptr_t>(wordPointer)));
119         dataLogF("%s%16s: 0x%016llx\n", prefix, buf, static_cast<long long>(*wordPointer));
120     }
121 }
122
123 template<typename DescriptorType>
124 void generateICFastPath(
125     State& state, CodeBlock* codeBlock, GeneratedFunction generatedFunction,
126     StackMaps::RecordMap& recordMap, DescriptorType& ic, size_t sizeOfIC)
127 {
128     VM& vm = state.graph.m_vm;
129
130     StackMaps::RecordMap::iterator iter = recordMap.find(ic.stackmapID());
131     if (iter == recordMap.end()) {
132         // It was optimized out.
133         return;
134     }
135     
136     Vector<StackMaps::Record>& records = iter->value;
137     
138     RELEASE_ASSERT(records.size() == ic.m_generators.size());
139     
140     for (unsigned i = records.size(); i--;) {
141         StackMaps::Record& record = records[i];
142         auto generator = ic.m_generators[i];
143
144         CCallHelpers fastPathJIT(&vm, codeBlock);
145         generator.generateFastPath(fastPathJIT);
146         
147         char* startOfIC =
148             bitwise_cast<char*>(generatedFunction) + record.instructionOffset;
149         
150         LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfIC);
151         // Note: we could handle the !isValid() case. We just don't appear to have a
152         // reason to do so, yet.
153         RELEASE_ASSERT(linkBuffer.isValid());
154         
155         MacroAssembler::AssemblerType_T::fillNops(
156             startOfIC + linkBuffer.size(), sizeOfIC - linkBuffer.size());
157         
158         state.finalizer->sideCodeLinkBuffer->link(
159             ic.m_slowPathDone[i], CodeLocationLabel(startOfIC + sizeOfIC));
160         
161         linkBuffer.link(
162             generator.slowPathJump(),
163             state.finalizer->sideCodeLinkBuffer->locationOf(generator.slowPathBegin()));
164         
165         generator.finalize(linkBuffer, *state.finalizer->sideCodeLinkBuffer);
166     }
167 }
168
169 static RegisterSet usedRegistersFor(const StackMaps::Record& record)
170 {
171     if (Options::assumeAllRegsInFTLICAreLive())
172         return RegisterSet::allRegisters();
173     return RegisterSet(record.usedRegisterSet(), RegisterSet::calleeSaveRegisters());
174 }
175
176 static void fixFunctionBasedOnStackMaps(
177     State& state, CodeBlock* codeBlock, JITCode* jitCode, GeneratedFunction generatedFunction,
178     StackMaps::RecordMap& recordMap, bool didSeeUnwindInfo)
179 {
180     Graph& graph = state.graph;
181     VM& vm = graph.m_vm;
182     StackMaps stackmaps = jitCode->stackmaps;
183     
184     StackMaps::RecordMap::iterator iter = recordMap.find(state.capturedStackmapID);
185     RELEASE_ASSERT(iter != recordMap.end());
186     RELEASE_ASSERT(iter->value.size() == 1);
187     RELEASE_ASSERT(iter->value[0].locations.size() == 1);
188     Location capturedLocation =
189         Location::forStackmaps(&jitCode->stackmaps, iter->value[0].locations[0]);
190     RELEASE_ASSERT(capturedLocation.kind() == Location::Register);
191     RELEASE_ASSERT(capturedLocation.gpr() == GPRInfo::callFrameRegister);
192     RELEASE_ASSERT(!(capturedLocation.addend() % sizeof(Register)));
193     int32_t localsOffset = capturedLocation.addend() / sizeof(Register) + graph.m_nextMachineLocal;
194     
195     for (unsigned i = graph.m_inlineVariableData.size(); i--;) {
196         InlineCallFrame* inlineCallFrame = graph.m_inlineVariableData[i].inlineCallFrame;
197         
198         if (inlineCallFrame->argumentsRegister.isValid()) {
199             inlineCallFrame->argumentsRegister = VirtualRegister(
200                 inlineCallFrame->argumentsRegister.offset() + localsOffset);
201         }
202         
203         for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
204             inlineCallFrame->arguments[argument] =
205                 inlineCallFrame->arguments[argument].withLocalsOffset(localsOffset);
206         }
207         
208         if (inlineCallFrame->isClosureCall) {
209             inlineCallFrame->calleeRecovery =
210                 inlineCallFrame->calleeRecovery.withLocalsOffset(localsOffset);
211         }
212     }
213     
214     if (codeBlock->usesArguments()) {
215         codeBlock->setArgumentsRegister(
216             VirtualRegister(codeBlock->argumentsRegister().offset() + localsOffset));
217     }
218
219     MacroAssembler::Label stackOverflowException;
220
221     {
222         CCallHelpers checkJIT(&vm, codeBlock);
223         
224         // At this point it's perfectly fair to just blow away all state and restore the
225         // JS JIT view of the universe.
226         checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
227
228         MacroAssembler::Label exceptionContinueArg1Set = checkJIT.label();
229         checkJIT.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
230         checkJIT.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
231
232         checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0);
233         MacroAssembler::Call call = checkJIT.call();
234         checkJIT.jumpToExceptionHandler();
235
236         stackOverflowException = checkJIT.label();
237         checkJIT.emitGetCallerFrameFromCallFrameHeaderPtr(GPRInfo::argumentGPR1);
238         checkJIT.jump(exceptionContinueArg1Set);
239
240         OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(
241             vm, checkJIT, codeBlock, JITCompilationMustSucceed));
242         linkBuffer->link(call, FunctionPtr(lookupExceptionHandler));
243         
244         state.finalizer->handleExceptionsLinkBuffer = linkBuffer.release();
245     }
246
247     ExitThunkGenerator exitThunkGenerator(state);
248     exitThunkGenerator.emitThunks();
249     if (exitThunkGenerator.didThings()) {
250         RELEASE_ASSERT(state.finalizer->osrExit.size());
251         RELEASE_ASSERT(didSeeUnwindInfo);
252         
253         OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(
254             vm, exitThunkGenerator, codeBlock, JITCompilationMustSucceed));
255         
256         RELEASE_ASSERT(state.finalizer->osrExit.size() == state.jitCode->osrExit.size());
257         
258         for (unsigned i = 0; i < state.jitCode->osrExit.size(); ++i) {
259             OSRExitCompilationInfo& info = state.finalizer->osrExit[i];
260             OSRExit& exit = jitCode->osrExit[i];
261             
262             if (verboseCompilationEnabled())
263                 dataLog("Handling OSR stackmap #", exit.m_stackmapID, " for ", exit.m_codeOrigin, "\n");
264
265             iter = recordMap.find(exit.m_stackmapID);
266             if (iter == recordMap.end()) {
267                 // It was optimized out.
268                 continue;
269             }
270             
271             info.m_thunkAddress = linkBuffer->locationOf(info.m_thunkLabel);
272             exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump);
273             
274             for (unsigned j = exit.m_values.size(); j--;) {
275                 ExitValue value = exit.m_values[j];
276                 if (!value.isInJSStackSomehow())
277                     continue;
278                 if (!value.virtualRegister().isLocal())
279                     continue;
280                 exit.m_values[j] = value.withVirtualRegister(
281                     VirtualRegister(value.virtualRegister().offset() + localsOffset));
282             }
283             
284             if (verboseCompilationEnabled()) {
285                 DumpContext context;
286                 dataLog("    Exit values: ", inContext(exit.m_values, &context), "\n");
287             }
288         }
289         
290         state.finalizer->exitThunksLinkBuffer = linkBuffer.release();
291     }
292
293     if (!state.getByIds.isEmpty() || !state.putByIds.isEmpty()) {
294         CCallHelpers slowPathJIT(&vm, codeBlock);
295         
296         CCallHelpers::JumpList exceptionTarget;
297         
298         for (unsigned i = state.getByIds.size(); i--;) {
299             GetByIdDescriptor& getById = state.getByIds[i];
300             
301             if (verboseCompilationEnabled())
302                 dataLog("Handling GetById stackmap #", getById.stackmapID(), "\n");
303             
304             iter = recordMap.find(getById.stackmapID());
305             if (iter == recordMap.end()) {
306                 // It was optimized out.
307                 continue;
308             }
309             
310             for (unsigned i = 0; i < iter->value.size(); ++i) {
311                 StackMaps::Record& record = iter->value[i];
312             
313                 RegisterSet usedRegisters = usedRegistersFor(record);
314                 
315                 GPRReg result = record.locations[0].directGPR();
316                 GPRReg base = record.locations[1].directGPR();
317                 
318                 JITGetByIdGenerator gen(
319                     codeBlock, getById.codeOrigin(), usedRegisters, JSValueRegs(base),
320                     JSValueRegs(result), NeedToSpill);
321                 
322                 MacroAssembler::Label begin = slowPathJIT.label();
323                 
324                 MacroAssembler::Call call = callOperation(
325                     state, usedRegisters, slowPathJIT, getById.codeOrigin(), &exceptionTarget,
326                     operationGetByIdOptimize, result, gen.stubInfo(), base, getById.uid());
327                 
328                 gen.reportSlowPathCall(begin, call);
329                 
330                 getById.m_slowPathDone.append(slowPathJIT.jump());
331                 getById.m_generators.append(gen);
332             }
333         }
334         
335         for (unsigned i = state.putByIds.size(); i--;) {
336             PutByIdDescriptor& putById = state.putByIds[i];
337             
338             if (verboseCompilationEnabled())
339                 dataLog("Handling PutById stackmap #", putById.stackmapID(), "\n");
340             
341             iter = recordMap.find(putById.stackmapID());
342             if (iter == recordMap.end()) {
343                 // It was optimized out.
344                 continue;
345             }
346             
347             for (unsigned i = 0; i < iter->value.size(); ++i) {
348                 StackMaps::Record& record = iter->value[i];
349                 
350                 RegisterSet usedRegisters = usedRegistersFor(record);
351                 
352                 GPRReg base = record.locations[0].directGPR();
353                 GPRReg value = record.locations[1].directGPR();
354                 
355                 JITPutByIdGenerator gen(
356                     codeBlock, putById.codeOrigin(), usedRegisters, JSValueRegs(base),
357                     JSValueRegs(value), GPRInfo::patchpointScratchRegister, NeedToSpill,
358                     putById.ecmaMode(), putById.putKind());
359                 
360                 MacroAssembler::Label begin = slowPathJIT.label();
361                 
362                 MacroAssembler::Call call = callOperation(
363                     state, usedRegisters, slowPathJIT, putById.codeOrigin(), &exceptionTarget,
364                     gen.slowPathFunction(), gen.stubInfo(), value, base, putById.uid());
365                 
366                 gen.reportSlowPathCall(begin, call);
367                 
368                 putById.m_slowPathDone.append(slowPathJIT.jump());
369                 putById.m_generators.append(gen);
370             }
371         }
372         
373         exceptionTarget.link(&slowPathJIT);
374         MacroAssembler::Jump exceptionJump = slowPathJIT.jump();
375         
376         state.finalizer->sideCodeLinkBuffer = adoptPtr(
377             new LinkBuffer(vm, slowPathJIT, codeBlock, JITCompilationMustSucceed));
378         state.finalizer->sideCodeLinkBuffer->link(
379             exceptionJump, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
380         
381         for (unsigned i = state.getByIds.size(); i--;) {
382             generateICFastPath(
383                 state, codeBlock, generatedFunction, recordMap, state.getByIds[i],
384                 sizeOfGetById());
385         }
386         for (unsigned i = state.putByIds.size(); i--;) {
387             generateICFastPath(
388                 state, codeBlock, generatedFunction, recordMap, state.putByIds[i],
389                 sizeOfPutById());
390         }
391     }
392     
393     // Handling JS calls is weird: we need to ensure that we sort them by the PC in LLVM
394     // generated code. That implies first pruning the ones that LLVM didn't generate.
395     Vector<JSCall> oldCalls = state.jsCalls;
396     state.jsCalls.resize(0);
397     for (unsigned i = 0; i < oldCalls.size(); ++i) {
398         JSCall& call = oldCalls[i];
399         
400         StackMaps::RecordMap::iterator iter = recordMap.find(call.stackmapID());
401         if (iter == recordMap.end())
402             continue;
403
404         for (unsigned j = 0; j < iter->value.size(); ++j) {
405             JSCall copy = call;
406             copy.m_instructionOffset = iter->value[j].instructionOffset;
407             state.jsCalls.append(copy);
408         }
409     }
410     
411     std::sort(state.jsCalls.begin(), state.jsCalls.end());
412     
413     for (unsigned i = state.jsCalls.size(); i--;) {
414         JSCall& call = state.jsCalls[i];
415
416         CCallHelpers fastPathJIT(&vm, codeBlock);
417         call.emit(fastPathJIT);
418         
419         char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset;
420         
421         LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfCall());
422         if (!linkBuffer.isValid()) {
423             dataLog("Failed to insert inline cache for call because we thought the size would be ", sizeOfCall(), " but it ended up being ", fastPathJIT.m_assembler.codeSize(), " prior to compaction.\n");
424             RELEASE_ASSERT_NOT_REACHED();
425         }
426         
427         MacroAssembler::AssemblerType_T::fillNops(
428             startOfIC + linkBuffer.size(), sizeOfCall() - linkBuffer.size());
429         
430         call.link(vm, linkBuffer);
431     }
432     
433     RepatchBuffer repatchBuffer(codeBlock);
434
435     iter = recordMap.find(state.handleStackOverflowExceptionStackmapID);
436     // It's sort of remotely possible that we won't have an in-band exception handling
437     // path, for some kinds of functions.
438     if (iter != recordMap.end()) {
439         for (unsigned i = iter->value.size(); i--;) {
440             StackMaps::Record& record = iter->value[i];
441             
442             CodeLocationLabel source = CodeLocationLabel(
443                 bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
444
445             RELEASE_ASSERT(stackOverflowException.isSet());
446
447             repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->locationOf(stackOverflowException));
448         }
449     }
450     
451     iter = recordMap.find(state.handleExceptionStackmapID);
452     // It's sort of remotely possible that we won't have an in-band exception handling
453     // path, for some kinds of functions.
454     if (iter != recordMap.end()) {
455         for (unsigned i = iter->value.size(); i--;) {
456             StackMaps::Record& record = iter->value[i];
457             
458             CodeLocationLabel source = CodeLocationLabel(
459                 bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
460             
461             repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
462         }
463     }
464     
465     for (unsigned exitIndex = 0; exitIndex < jitCode->osrExit.size(); ++exitIndex) {
466         OSRExitCompilationInfo& info = state.finalizer->osrExit[exitIndex];
467         OSRExit& exit = jitCode->osrExit[exitIndex];
468         iter = recordMap.find(exit.m_stackmapID);
469         
470         Vector<const void*> codeAddresses;
471         
472         if (iter != recordMap.end()) {
473             for (unsigned i = iter->value.size(); i--;) {
474                 StackMaps::Record& record = iter->value[i];
475                 
476                 CodeLocationLabel source = CodeLocationLabel(
477                     bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
478                 
479                 codeAddresses.append(bitwise_cast<char*>(generatedFunction) + record.instructionOffset + MacroAssembler::maxJumpReplacementSize());
480                 
481                 if (info.m_isInvalidationPoint)
482                     jitCode->common.jumpReplacements.append(JumpReplacement(source, info.m_thunkAddress));
483                 else
484                     repatchBuffer.replaceWithJump(source, info.m_thunkAddress);
485             }
486         }
487         
488         if (graph.compilation())
489             graph.compilation()->addOSRExitSite(codeAddresses);
490     }
491 }
492
493 void compile(State& state, Safepoint::Result& safepointResult)
494 {
495     char* error = 0;
496     
497     {
498         GraphSafepoint safepoint(state.graph, safepointResult);
499         
500         LLVMMCJITCompilerOptions options;
501         llvm->InitializeMCJITCompilerOptions(&options, sizeof(options));
502         options.OptLevel = Options::llvmBackendOptimizationLevel();
503         options.NoFramePointerElim = true;
504         if (Options::useLLVMSmallCodeModel())
505             options.CodeModel = LLVMCodeModelSmall;
506         options.EnableFastISel = Options::enableLLVMFastISel();
507         options.MCJMM = llvm->CreateSimpleMCJITMemoryManager(
508             &state, mmAllocateCodeSection, mmAllocateDataSection, mmApplyPermissions, mmDestroy);
509     
510         LLVMExecutionEngineRef engine;
511         
512         if (isARM64())
513             llvm->SetTarget(state.module, "arm64-apple-ios");
514         
515         if (llvm->CreateMCJITCompilerForModule(&engine, state.module, &options, sizeof(options), &error)) {
516             dataLog("FATAL: Could not create LLVM execution engine: ", error, "\n");
517             CRASH();
518         }
519
520         LLVMPassManagerRef functionPasses = 0;
521         LLVMPassManagerRef modulePasses;
522     
523         if (Options::llvmSimpleOpt()) {
524             modulePasses = llvm->CreatePassManager();
525             llvm->AddTargetData(llvm->GetExecutionEngineTargetData(engine), modulePasses);
526
527             LLVMTargetMachineRef targetMachine = llvm->GetExecutionEngineTargetMachine(engine);
528              
529             llvm->AddAnalysisPasses(targetMachine, modulePasses);
530             llvm->AddPromoteMemoryToRegisterPass(modulePasses);
531  
532             llvm->AddGlobalOptimizerPass(modulePasses);
533  
534             llvm->AddFunctionInliningPass(modulePasses);
535             llvm->AddPruneEHPass(modulePasses);
536             llvm->AddGlobalDCEPass(modulePasses);
537              
538             llvm->AddConstantPropagationPass(modulePasses);
539             llvm->AddAggressiveDCEPass(modulePasses);
540             llvm->AddInstructionCombiningPass(modulePasses);
541             llvm->AddBasicAliasAnalysisPass(modulePasses);
542             llvm->AddTypeBasedAliasAnalysisPass(modulePasses);
543             llvm->AddGVNPass(modulePasses);
544             llvm->AddCFGSimplificationPass(modulePasses);
545             llvm->AddDeadStoreEliminationPass(modulePasses);
546  
547             llvm->RunPassManager(modulePasses, state.module);
548         } else {
549             LLVMPassManagerBuilderRef passBuilder = llvm->PassManagerBuilderCreate();
550             llvm->PassManagerBuilderSetOptLevel(passBuilder, Options::llvmOptimizationLevel());
551             llvm->PassManagerBuilderUseInlinerWithThreshold(passBuilder, 275);
552             llvm->PassManagerBuilderSetSizeLevel(passBuilder, Options::llvmSizeLevel());
553         
554             functionPasses = llvm->CreateFunctionPassManagerForModule(state.module);
555             modulePasses = llvm->CreatePassManager();
556         
557             llvm->AddTargetData(llvm->GetExecutionEngineTargetData(engine), modulePasses);
558         
559             llvm->PassManagerBuilderPopulateFunctionPassManager(passBuilder, functionPasses);
560             llvm->PassManagerBuilderPopulateModulePassManager(passBuilder, modulePasses);
561         
562             llvm->PassManagerBuilderDispose(passBuilder);
563         
564             llvm->InitializeFunctionPassManager(functionPasses);
565             for (LValue function = llvm->GetFirstFunction(state.module); function; function = llvm->GetNextFunction(function))
566                 llvm->RunFunctionPassManager(functionPasses, function);
567             llvm->FinalizeFunctionPassManager(functionPasses);
568         
569             llvm->RunPassManager(modulePasses, state.module);
570         }
571
572         if (shouldShowDisassembly() || verboseCompilationEnabled())
573             state.dumpState("after optimization");
574     
575         // FIXME: Need to add support for the case where JIT memory allocation failed.
576         // https://bugs.webkit.org/show_bug.cgi?id=113620
577         state.generatedFunction = reinterpret_cast<GeneratedFunction>(llvm->GetPointerToGlobal(engine, state.function));
578         if (functionPasses)
579             llvm->DisposePassManager(functionPasses);
580         llvm->DisposePassManager(modulePasses);
581         llvm->DisposeExecutionEngine(engine);
582     }
583     if (safepointResult.didGetCancelled())
584         return;
585     RELEASE_ASSERT(!state.graph.m_vm.heap.isCollecting());
586     
587     if (shouldShowDisassembly()) {
588         for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) {
589             ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get();
590             dataLog(
591                 "Generated LLVM code for ",
592                 CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT),
593                 " #", i, ", ", state.codeSectionNames[i], ":\n");
594             disassemble(
595                 MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(),
596                 "    ", WTF::dataFile(), LLVMSubset);
597         }
598         
599         for (unsigned i = 0; i < state.jitCode->dataSections().size(); ++i) {
600             DataSection* section = state.jitCode->dataSections()[i].get();
601             dataLog(
602                 "Generated LLVM data section for ",
603                 CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT),
604                 " #", i, ", ", state.dataSectionNames[i], ":\n");
605             dumpDataSection(section, "    ");
606         }
607     }
608     
609     bool didSeeUnwindInfo = state.jitCode->unwindInfo.parse(
610         state.compactUnwind, state.compactUnwindSize, state.generatedFunction);
611     if (shouldShowDisassembly()) {
612         dataLog("Unwind info for ", CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT), ":\n");
613         if (didSeeUnwindInfo)
614             dataLog("    ", state.jitCode->unwindInfo, "\n");
615         else
616             dataLog("    <no unwind info>\n");
617     }
618     
619     if (state.stackmapsSection && state.stackmapsSection->size()) {
620         if (shouldShowDisassembly()) {
621             dataLog(
622                 "Generated LLVM stackmaps section for ",
623                 CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT), ":\n");
624             dataLog("    Raw data:\n");
625             dumpDataSection(state.stackmapsSection.get(), "    ");
626         }
627         
628         RefPtr<DataView> stackmapsData = DataView::create(
629             ArrayBuffer::create(state.stackmapsSection->base(), state.stackmapsSection->size()));
630         state.jitCode->stackmaps.parse(stackmapsData.get());
631     
632         if (shouldShowDisassembly()) {
633             dataLog("    Structured data:\n");
634             state.jitCode->stackmaps.dumpMultiline(WTF::dataFile(), "        ");
635         }
636         
637         StackMaps::RecordMap recordMap = state.jitCode->stackmaps.computeRecordMap();
638         fixFunctionBasedOnStackMaps(
639             state, state.graph.m_codeBlock, state.jitCode.get(), state.generatedFunction,
640             recordMap, didSeeUnwindInfo);
641         
642         if (shouldShowDisassembly()) {
643             for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) {
644                 if (state.codeSectionNames[i] != "__text")
645                     continue;
646                 
647                 ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get();
648                 dataLog(
649                     "Generated LLVM code after stackmap-based fix-up for ",
650                     CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT),
651                     " in ", state.graph.m_plan.mode, " #", i, ", ",
652                     state.codeSectionNames[i], ":\n");
653                 disassemble(
654                     MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(),
655                     "    ", WTF::dataFile(), LLVMSubset);
656             }
657         }
658     }
659     
660     state.module = 0; // We no longer own the module.
661 }
662
663 } } // namespace JSC::FTL
664
665 #endif // ENABLE(FTL_JIT)
666