b236a5c802cb776a2cadbaa5ebcfe6bd1fb3a3f5
[WebKit-https.git] / Source / JavaScriptCore / assembler / LinkBuffer.cpp
1 /*
2  * Copyright (C) 2012-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "LinkBuffer.h"
28
29 #if ENABLE(ASSEMBLER)
30
31 #include "CodeBlock.h"
32 #include "Disassembler.h"
33 #include "JITCode.h"
34 #include "JSCInlines.h"
35 #include "Options.h"
36 #include <wtf/CompilationThread.h>
37
38 #if OS(LINUX)
39 #include "PerfLog.h"
40 #endif
41
42 namespace JSC {
43
44 bool shouldDumpDisassemblyFor(CodeBlock* codeBlock)
45 {
46     if (codeBlock && JITCode::isOptimizingJIT(codeBlock->jitType()) && Options::dumpDFGDisassembly())
47         return true;
48     return Options::dumpDisassembly();
49 }
50
51 LinkBuffer::CodeRef<LinkBufferPtrTag> LinkBuffer::finalizeCodeWithoutDisassemblyImpl()
52 {
53     performFinalization();
54     
55     ASSERT(m_didAllocate);
56     if (m_executableMemory)
57         return CodeRef<LinkBufferPtrTag>(*m_executableMemory);
58     
59     return CodeRef<LinkBufferPtrTag>::createSelfManagedCodeRef(m_code);
60 }
61
62 LinkBuffer::CodeRef<LinkBufferPtrTag> LinkBuffer::finalizeCodeWithDisassemblyImpl(bool dumpDisassembly, const char* format, ...)
63 {
64     CodeRef<LinkBufferPtrTag> result = finalizeCodeWithoutDisassemblyImpl();
65
66 #if OS(LINUX)
67     if (Options::logJITCodeForPerf()) {
68         StringPrintStream out;
69         va_list argList;
70         va_start(argList, format);
71         va_start(argList, format);
72         out.vprintf(format, argList);
73         va_end(argList);
74         PerfLog::log(out.toCString(), result.code().untaggedExecutableAddress<const uint8_t*>(), result.size());
75     }
76 #endif
77
78     if (!dumpDisassembly || m_alreadyDisassembled)
79         return result;
80     
81     StringPrintStream out;
82     out.printf("Generated JIT code for ");
83     va_list argList;
84     va_start(argList, format);
85     out.vprintf(format, argList);
86     va_end(argList);
87     out.printf(":\n");
88
89     uint8_t* executableAddress = result.code().untaggedExecutableAddress<uint8_t*>();
90     out.printf("    Code at [%p, %p):\n", executableAddress, executableAddress + result.size());
91     
92     CString header = out.toCString();
93     
94     if (Options::asyncDisassembly()) {
95         CodeRef<DisassemblyPtrTag> codeRefForDisassembly = result.retagged<DisassemblyPtrTag>();
96         disassembleAsynchronously(header, WTFMove(codeRefForDisassembly), m_size, "    ");
97         return result;
98     }
99     
100     dataLog(header);
101     disassemble(result.retaggedCode<DisassemblyPtrTag>(), m_size, "    ", WTF::dataFile());
102     
103     return result;
104 }
105
106 #if ENABLE(BRANCH_COMPACTION)
107 static ALWAYS_INLINE void recordLinkOffsets(AssemblerData& assemblerData, int32_t regionStart, int32_t regionEnd, int32_t offset)
108 {
109     int32_t ptr = regionStart / sizeof(int32_t);
110     const int32_t end = regionEnd / sizeof(int32_t);
111     int32_t* offsets = reinterpret_cast_ptr<int32_t*>(assemblerData.buffer());
112     while (ptr < end)
113         offsets[ptr++] = offset;
114 }
115
116 template <typename InstructionType>
117 void LinkBuffer::copyCompactAndLinkCode(MacroAssembler& macroAssembler, void* ownerUID, JITCompilationEffort effort)
118 {
119     allocate(macroAssembler, ownerUID, effort);
120     const size_t initialSize = macroAssembler.m_assembler.codeSize();
121     if (didFailToAllocate())
122         return;
123
124     Vector<LinkRecord, 0, UnsafeVectorOverflow>& jumpsToLink = macroAssembler.jumpsToLink();
125     m_assemblerStorage = macroAssembler.m_assembler.buffer().releaseAssemblerData();
126     uint8_t* inData = reinterpret_cast<uint8_t*>(m_assemblerStorage.buffer());
127
128     uint8_t* codeOutData = m_code.dataLocation<uint8_t*>();
129 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
130     const ARM64EHash assemblerBufferHash = macroAssembler.m_assembler.buffer().hash();
131     ARM64EHash verifyUncompactedHash(assemblerBufferHash.randomSeed());
132     uint8_t* outData = codeOutData;
133 #else
134     AssemblerData outBuffer(m_size);
135     uint8_t* outData = reinterpret_cast<uint8_t*>(outBuffer.buffer());
136 #endif
137 #if CPU(ARM64)
138     RELEASE_ASSERT(roundUpToMultipleOf<sizeof(unsigned)>(outData) == outData);
139     RELEASE_ASSERT(roundUpToMultipleOf<sizeof(unsigned)>(codeOutData) == codeOutData);
140 #endif
141
142     int readPtr = 0;
143     int writePtr = 0;
144     unsigned jumpCount = jumpsToLink.size();
145
146 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
147     os_thread_self_restrict_rwx_to_rw();
148 #endif
149
150     if (m_shouldPerformBranchCompaction) {
151         for (unsigned i = 0; i < jumpCount; ++i) {
152             int offset = readPtr - writePtr;
153             ASSERT(!(offset & 1));
154                 
155             // Copy the instructions from the last jump to the current one.
156             size_t regionSize = jumpsToLink[i].from() - readPtr;
157             InstructionType* copySource = reinterpret_cast_ptr<InstructionType*>(inData + readPtr);
158             InstructionType* copyEnd = reinterpret_cast_ptr<InstructionType*>(inData + readPtr + regionSize);
159             InstructionType* copyDst = reinterpret_cast_ptr<InstructionType*>(outData + writePtr);
160             ASSERT(!(regionSize % 2));
161             ASSERT(!(readPtr % 2));
162             ASSERT(!(writePtr % 2));
163 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
164             unsigned index = readPtr;
165 #endif
166             while (copySource != copyEnd) {
167                 InstructionType insn = *copySource++;
168 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
169                 static_assert(sizeof(InstructionType) == 4, "");
170                 verifyUncompactedHash.update(insn, index);
171                 index += sizeof(InstructionType);
172 #endif
173                 *copyDst++ = insn;
174             }
175             recordLinkOffsets(m_assemblerStorage, readPtr, jumpsToLink[i].from(), offset);
176             readPtr += regionSize;
177             writePtr += regionSize;
178                 
179             // Calculate absolute address of the jump target, in the case of backwards
180             // branches we need to be precise, forward branches we are pessimistic
181             const uint8_t* target;
182             if (jumpsToLink[i].to() >= jumpsToLink[i].from())
183                 target = codeOutData + jumpsToLink[i].to() - offset; // Compensate for what we have collapsed so far
184             else
185                 target = codeOutData + jumpsToLink[i].to() - executableOffsetFor(jumpsToLink[i].to());
186                 
187             JumpLinkType jumpLinkType = MacroAssembler::computeJumpType(jumpsToLink[i], codeOutData + writePtr, target);
188             // Compact branch if we can...
189             if (MacroAssembler::canCompact(jumpsToLink[i].type())) {
190                 // Step back in the write stream
191                 int32_t delta = MacroAssembler::jumpSizeDelta(jumpsToLink[i].type(), jumpLinkType);
192                 if (delta) {
193                     writePtr -= delta;
194                     recordLinkOffsets(m_assemblerStorage, jumpsToLink[i].from() - delta, readPtr, readPtr - writePtr);
195                 }
196             }
197             jumpsToLink[i].setFrom(writePtr);
198         }
199     } else {
200         if (!ASSERT_DISABLED) {
201             for (unsigned i = 0; i < jumpCount; ++i)
202                 ASSERT(!MacroAssembler::canCompact(jumpsToLink[i].type()));
203         }
204     }
205
206     // Copy everything after the last jump
207     {
208         InstructionType* dst = bitwise_cast<InstructionType*>(outData + writePtr);
209         InstructionType* src = bitwise_cast<InstructionType*>(inData + readPtr);
210         size_t bytes = initialSize - readPtr;
211
212         RELEASE_ASSERT(bitwise_cast<uintptr_t>(dst) % sizeof(InstructionType) == 0);
213         RELEASE_ASSERT(bitwise_cast<uintptr_t>(src) % sizeof(InstructionType) == 0);
214         RELEASE_ASSERT(bytes % sizeof(InstructionType) == 0);
215
216 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
217         unsigned index = readPtr;
218 #endif
219
220         for (size_t i = 0; i < bytes; i += sizeof(InstructionType)) {
221             InstructionType insn = *src++;
222 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
223             verifyUncompactedHash.update(insn, index);
224             index += sizeof(InstructionType);
225 #endif
226             *dst++ = insn;
227         }
228     }
229
230 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
231     if (verifyUncompactedHash.hash() != assemblerBufferHash.hash()) {
232         dataLogLn("Hashes don't match: ", RawPointer(bitwise_cast<void*>(verifyUncompactedHash.hash())), " ", RawPointer(bitwise_cast<void*>(assemblerBufferHash.hash())));
233         dataLogLn("Crashing!");
234         CRASH();
235     }
236 #endif
237
238     recordLinkOffsets(m_assemblerStorage, readPtr, initialSize, readPtr - writePtr);
239         
240     for (unsigned i = 0; i < jumpCount; ++i) {
241 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
242         auto memcpyFunction = memcpy;
243 #else
244         auto memcpyFunction = performJITMemcpy;
245 #endif
246
247         uint8_t* location = codeOutData + jumpsToLink[i].from();
248         uint8_t* target = codeOutData + jumpsToLink[i].to() - executableOffsetFor(jumpsToLink[i].to());
249         MacroAssembler::link(jumpsToLink[i], outData + jumpsToLink[i].from(), location, target, memcpyFunction);
250     }
251
252     size_t compactSize = writePtr + initialSize - readPtr;
253     if (!m_executableMemory) {
254         size_t nopSizeInBytes = initialSize - compactSize;
255         MacroAssembler::AssemblerType_T::fillNops(outData + compactSize, nopSizeInBytes, memcpy);
256     }
257
258 #if CPU(ARM64E) && ENABLE(FAST_JIT_PERMISSIONS)
259     os_thread_self_restrict_rwx_to_rx();
260 #endif
261
262     if (m_executableMemory) {
263         m_size = compactSize;
264         m_executableMemory->shrink(m_size);
265     }
266
267 #if !CPU(ARM64E) || !ENABLE(FAST_JIT_PERMISSIONS)
268     ASSERT(codeOutData != outData);
269     performJITMemcpy(codeOutData, outData, m_size);
270 #else
271     ASSERT(codeOutData == outData);
272 #endif
273
274     jumpsToLink.clear();
275
276 #if DUMP_LINK_STATISTICS
277     dumpLinkStatistics(codeOutData, initialSize, m_size);
278 #endif
279 #if DUMP_CODE
280     dumpCode(codeOutData, m_size);
281 #endif
282 }
283 #endif
284
285
286 void LinkBuffer::linkCode(MacroAssembler& macroAssembler, void* ownerUID, JITCompilationEffort effort)
287 {
288     // Ensure that the end of the last invalidation point does not extend beyond the end of the buffer.
289     macroAssembler.label();
290
291 #if !ENABLE(BRANCH_COMPACTION)
292 #if defined(ASSEMBLER_HAS_CONSTANT_POOL) && ASSEMBLER_HAS_CONSTANT_POOL
293     macroAssembler.m_assembler.buffer().flushConstantPool(false);
294 #endif
295     allocate(macroAssembler, ownerUID, effort);
296     if (!m_didAllocate)
297         return;
298     ASSERT(m_code);
299     AssemblerBuffer& buffer = macroAssembler.m_assembler.buffer();
300     void* code = m_code.dataLocation();
301 #if CPU(ARM64)
302     RELEASE_ASSERT(roundUpToMultipleOf<Assembler::instructionSize>(code) == code);
303 #endif
304     performJITMemcpy(code, buffer.data(), buffer.codeSize());
305 #if CPU(MIPS)
306     macroAssembler.m_assembler.relocateJumps(buffer.data(), code);
307 #endif
308 #elif CPU(ARM_THUMB2)
309     copyCompactAndLinkCode<uint16_t>(macroAssembler, ownerUID, effort);
310 #elif CPU(ARM64)
311     copyCompactAndLinkCode<uint32_t>(macroAssembler, ownerUID, effort);
312 #endif // !ENABLE(BRANCH_COMPACTION)
313
314     m_linkTasks = WTFMove(macroAssembler.m_linkTasks);
315 }
316
317 void LinkBuffer::allocate(MacroAssembler& macroAssembler, void* ownerUID, JITCompilationEffort effort)
318 {
319     size_t initialSize = macroAssembler.m_assembler.codeSize();
320     if (m_code) {
321         if (initialSize > m_size)
322             return;
323         
324         size_t nopsToFillInBytes = m_size - initialSize;
325         macroAssembler.emitNops(nopsToFillInBytes);
326         m_didAllocate = true;
327         return;
328     }
329     
330     while (initialSize % jitAllocationGranule) {
331         macroAssembler.breakpoint();
332         initialSize = macroAssembler.m_assembler.codeSize();
333     }
334
335     m_executableMemory = ExecutableAllocator::singleton().allocate(initialSize, ownerUID, effort);
336     if (!m_executableMemory)
337         return;
338     m_code = MacroAssemblerCodePtr<LinkBufferPtrTag>(m_executableMemory->start().retaggedPtr<LinkBufferPtrTag>());
339     m_size = initialSize;
340     m_didAllocate = true;
341 }
342
343 void LinkBuffer::performFinalization()
344 {
345     for (auto& task : m_linkTasks)
346         task->run(*this);
347
348 #ifndef NDEBUG
349     ASSERT(!isCompilationThread());
350     ASSERT(!m_completed);
351     ASSERT(isValid());
352     m_completed = true;
353 #endif
354     
355     MacroAssembler::cacheFlush(code(), m_size);
356 }
357
358 #if DUMP_LINK_STATISTICS
359 void LinkBuffer::dumpLinkStatistics(void* code, size_t initializeSize, size_t finalSize)
360 {
361     static unsigned linkCount = 0;
362     static unsigned totalInitialSize = 0;
363     static unsigned totalFinalSize = 0;
364     linkCount++;
365     totalInitialSize += initialSize;
366     totalFinalSize += finalSize;
367     dataLogF("link %p: orig %u, compact %u (delta %u, %.2f%%)\n", 
368             code, static_cast<unsigned>(initialSize), static_cast<unsigned>(finalSize),
369             static_cast<unsigned>(initialSize - finalSize),
370             100.0 * (initialSize - finalSize) / initialSize);
371     dataLogF("\ttotal %u: orig %u, compact %u (delta %u, %.2f%%)\n", 
372             linkCount, totalInitialSize, totalFinalSize, totalInitialSize - totalFinalSize,
373             100.0 * (totalInitialSize - totalFinalSize) / totalInitialSize);
374 }
375 #endif
376
377 #if DUMP_CODE
378 void LinkBuffer::dumpCode(void* code, size_t size)
379 {
380 #if CPU(ARM_THUMB2)
381     // Dump the generated code in an asm file format that can be assembled and then disassembled
382     // for debugging purposes. For example, save this output as jit.s:
383     //   gcc -arch armv7 -c jit.s
384     //   otool -tv jit.o
385     static unsigned codeCount = 0;
386     unsigned short* tcode = static_cast<unsigned short*>(code);
387     size_t tsize = size / sizeof(short);
388     char nameBuf[128];
389     snprintf(nameBuf, sizeof(nameBuf), "_jsc_jit%u", codeCount++);
390     dataLogF("\t.syntax unified\n"
391             "\t.section\t__TEXT,__text,regular,pure_instructions\n"
392             "\t.globl\t%s\n"
393             "\t.align 2\n"
394             "\t.code 16\n"
395             "\t.thumb_func\t%s\n"
396             "# %p\n"
397             "%s:\n", nameBuf, nameBuf, code, nameBuf);
398         
399     for (unsigned i = 0; i < tsize; i++)
400         dataLogF("\t.short\t0x%x\n", tcode[i]);
401 #endif
402 }
403 #endif
404
405 } // namespace JSC
406
407 #endif // ENABLE(ASSEMBLER)