Add the ability to change sp and pc to the ARM64 JIT probe.

https://bugs.webkit.org/show_bug.cgi?id=174697
<rdar://problem/33436965>

Reviewed by JF Bastien.

This patch implements the following:

1. The ARM64 probe now supports modifying the pc and sp.

   However, lr is not preserved when modifying the pc because it is used as the
   scratch register for the indirect jump. Hence, the probe handler function
   may not modify both lr and pc in the same probe invocation.

2. Fix probe tests to use bitwise comparison when comparing double register
   values. Otherwise, equivalent nan values will be interpreted as not equivalent.

3. Change the minimum offset increment in testProbeModifiesStackPointer to be
   16 bytes for ARM64.  This is because the ARM64 probe now uses the ldp and stp
   instructions which require 16 byte alignment for their memory access.

* assembler/MacroAssemblerARM64.cpp:
(JSC::arm64ProbeError):
(JSC::MacroAssembler::probe):
(JSC::arm64ProbeTrampoline): Deleted.
* assembler/testmasm.cpp:
(JSC::isSpecialGPR):
(JSC::testProbeReadsArgumentRegisters):
(JSC::testProbeWritesArgumentRegisters):
(JSC::testProbePreservesGPRS):
(JSC::testProbeModifiesStackPointer):
(JSC::testProbeModifiesStackPointerToInsideProbeContextOnStack):
(JSC::testProbeModifiesStackPointerToNBytesBelowSP):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@219951 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Source/JavaScriptCore/ChangeLog b/Source/JavaScriptCore/ChangeLog
index 9cbb18a..877283b 100644 (file)
--- a/Source/JavaScriptCore/ChangeLog
+++ b/Source/JavaScriptCore/ChangeLog
@@ -1,3 +1,39 @@
+2017-07-26  Mark Lam  <mark.lam@apple.com>
+
+        Add the ability to change sp and pc to the ARM64 JIT probe.
+        https://bugs.webkit.org/show_bug.cgi?id=174697
+        <rdar://problem/33436965>
+
+        Reviewed by JF Bastien.
+
+        This patch implements the following:
+
+        1. The ARM64 probe now supports modifying the pc and sp.
+
+           However, lr is not preserved when modifying the pc because it is used as the
+           scratch register for the indirect jump. Hence, the probe handler function
+           may not modify both lr and pc in the same probe invocation.
+
+        2. Fix probe tests to use bitwise comparison when comparing double register
+           values. Otherwise, equivalent nan values will be interpreted as not equivalent.
+
+        3. Change the minimum offset increment in testProbeModifiesStackPointer to be
+           16 bytes for ARM64.  This is because the ARM64 probe now uses the ldp and stp
+           instructions which require 16 byte alignment for their memory access.
+
+        * assembler/MacroAssemblerARM64.cpp:
+        (JSC::arm64ProbeError):
+        (JSC::MacroAssembler::probe):
+        (JSC::arm64ProbeTrampoline): Deleted.
+        * assembler/testmasm.cpp:
+        (JSC::isSpecialGPR):
+        (JSC::testProbeReadsArgumentRegisters):
+        (JSC::testProbeWritesArgumentRegisters):
+        (JSC::testProbePreservesGPRS):
+        (JSC::testProbeModifiesStackPointer):
+        (JSC::testProbeModifiesStackPointerToInsideProbeContextOnStack):
+        (JSC::testProbeModifiesStackPointerToNBytesBelowSP):
+
 2017-07-25  JF Bastien  <jfbastien@apple.com>
 
         WebAssembly: generate smaller binaries

diff --git a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.cpp b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.cpp
index 0a8b820..8c85786 100644 (file)
--- a/Source/JavaScriptCore/assembler/MacroAssemblerARM64.cpp
+++ b/Source/JavaScriptCore/assembler/MacroAssemblerARM64.cpp
@@ -122,9 +122,11 @@ using namespace ARM64Registers;
 #define PROBE_CPU_Q30_OFFSET (PROBE_FIRST_FPREG_OFFSET + (30 * FPREG_SIZE))
 #define PROBE_CPU_Q31_OFFSET (PROBE_FIRST_FPREG_OFFSET + (31 * FPREG_SIZE))
 #define PROBE_SIZE (PROBE_FIRST_FPREG_OFFSET + (32 * FPREG_SIZE))
-#define SAVED_CALLER_SP PROBE_SIZE
-#define PROBE_SIZE_PLUS_SAVED_CALLER_SP (SAVED_CALLER_SP + PTR_SIZE)
-#define PROBE_ALIGNED_SIZE (PROBE_SIZE_PLUS_SAVED_CALLER_SP)
+
+#define SAVED_PROBE_RETURN_PC_OFFSET        (PROBE_SIZE + (0 * PTR_SIZE))
+#define SAVED_PROBE_LR_OFFSET               (PROBE_SIZE + (1 * PTR_SIZE))
+#define SAVED_PROBE_ERROR_FUNCTION_OFFSET   (PROBE_SIZE + (2 * PTR_SIZE))
+#define PROBE_SIZE_PLUS_EXTRAS              (PROBE_SIZE + (3 * PTR_SIZE))
 
 // These ASSERTs remind you that if you change the layout of ProbeContext,
 // you need to change ctiMasmProbeTrampoline offsets above to match.
@@ -209,294 +211,345 @@ COMPILE_ASSERT(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q30]) == PROBE_CPU_Q30_OF
 COMPILE_ASSERT(PROBE_OFFSETOF(cpu.fprs[ARM64Registers::q31]) == PROBE_CPU_Q31_OFFSET, ProbeContext_cpu_q31_offset_matches_ctiMasmProbeTrampoline);
 
 COMPILE_ASSERT(sizeof(ProbeContext) == PROBE_SIZE, ProbeContext_size_matches_ctiMasmProbeTrampoline);
-COMPILE_ASSERT(!(PROBE_ALIGNED_SIZE & 0xf), ProbeContext_aligned_size_offset_should_be_16_byte_aligned);
+
+// Conditions for using ldp and stp.
+static_assert(PROBE_CPU_PC_OFFSET == PROBE_CPU_SP_OFFSET + PTR_SIZE, "PROBE_CPU_SP_OFFSET and PROBE_CPU_PC_OFFSET must be adjacent");
 
 #undef PROBE_OFFSETOF
 
+#define FPR_OFFSET(fpr) (PROBE_CPU_##fpr##_OFFSET - PROBE_CPU_Q0_OFFSET)
+
+struct IncomingProbeRecord {
+    uintptr_t probeHandlerFunction;
+    uintptr_t probeArg;
+    uintptr_t x26;
+    uintptr_t x27;
+    uintptr_t lr;
+    uintptr_t sp;
+    uintptr_t probeErrorFunction;
+    uintptr_t unused; // Padding for alignment.
+};
+
+#define IN_HANDLER_FUNCTION_OFFSET (0 * PTR_SIZE)
+#define IN_ARG_OFFSET              (1 * PTR_SIZE)
+#define IN_X26_OFFSET              (2 * PTR_SIZE)
+#define IN_X27_OFFSET              (3 * PTR_SIZE)
+#define IN_LR_OFFSET               (4 * PTR_SIZE)
+#define IN_SP_OFFSET               (5 * PTR_SIZE)
+#define IN_ERROR_FUNCTION_OFFSET   (6 * PTR_SIZE)
+
+static_assert(IN_HANDLER_FUNCTION_OFFSET == offsetof(IncomingProbeRecord, probeHandlerFunction), "IN_HANDLER_FUNCTION_OFFSET is incorrect");
+static_assert(IN_ARG_OFFSET == offsetof(IncomingProbeRecord, probeArg), "IN_ARG_OFFSET is incorrect");
+static_assert(IN_X26_OFFSET == offsetof(IncomingProbeRecord, x26), "IN_X26_OFFSET is incorrect");
+static_assert(IN_X27_OFFSET == offsetof(IncomingProbeRecord, x27), "IN_X27_OFFSET is incorrect");
+static_assert(IN_LR_OFFSET == offsetof(IncomingProbeRecord, lr), "IN_LR_OFFSET is incorrect");
+static_assert(IN_SP_OFFSET == offsetof(IncomingProbeRecord, sp), "IN_SP_OFFSET is incorrect");
+static_assert(IN_ERROR_FUNCTION_OFFSET == offsetof(IncomingProbeRecord, probeErrorFunction), "IN_ERROR_FUNCTION_OFFSET is incorrect");
+static_assert(!(sizeof(IncomingProbeRecord) & 0xf), "IncomingProbeStack must be 16-byte aligned");
+
+struct OutgoingProbeRecord {
+    uintptr_t nzcv;
+    uintptr_t fpsr;
+    uintptr_t x27;
+    uintptr_t x28;
+    uintptr_t fp;
+    uintptr_t lr;
+};
+
+#define OUT_NZCV_OFFSET (0 * PTR_SIZE)
+#define OUT_FPSR_OFFSET (1 * PTR_SIZE)
+#define OUT_X27_OFFSET  (2 * PTR_SIZE)
+#define OUT_X28_OFFSET  (3 * PTR_SIZE)
+#define OUT_FP_OFFSET   (4 * PTR_SIZE)
+#define OUT_LR_OFFSET   (5 * PTR_SIZE)
+#define OUT_SIZE        (6 * PTR_SIZE)
+
+static_assert(OUT_NZCV_OFFSET == offsetof(OutgoingProbeRecord, nzcv), "OUT_NZCV_OFFSET is incorrect");
+static_assert(OUT_FPSR_OFFSET == offsetof(OutgoingProbeRecord, fpsr), "OUT_FPSR_OFFSET is incorrect");
+static_assert(OUT_X27_OFFSET == offsetof(OutgoingProbeRecord, x27), "OUT_X27_OFFSET is incorrect");
+static_assert(OUT_X28_OFFSET == offsetof(OutgoingProbeRecord, x28), "OUT_X28_OFFSET is incorrect");
+static_assert(OUT_FP_OFFSET == offsetof(OutgoingProbeRecord, fp), "OUT_FP_OFFSET is incorrect");
+static_assert(OUT_LR_OFFSET == offsetof(OutgoingProbeRecord, lr), "OUT_LR_OFFSET is incorrect");
+static_assert(OUT_SIZE == sizeof(OutgoingProbeRecord), "OUT_SIZE is incorrect");
+static_assert(!(sizeof(OutgoingProbeRecord) & 0xf), "IncomingProbeStack must be 16-byte aligned");
+
+#define STATE_PC_NOT_CHANGED 0
+#define STATE_PC_CHANGED 1
+static_assert(STATE_PC_NOT_CHANGED != STATE_PC_CHANGED, "STATE_PC_NOT_CHANGED and STATE_PC_CHANGED should not be equal");
+
 asm (
     ".text" "\n"
-    ".align 2" "\n"
+    ".balign 16" "\n"
     ".globl " SYMBOL_STRING(ctiMasmProbeTrampoline) "\n"
     HIDE_SYMBOL(ctiMasmProbeTrampoline) "\n"
     SYMBOL_STRING(ctiMasmProbeTrampoline) ":" "\n"
 
-    // MacroAssemblerARM64::probe() has already generated code to store some values.
-    // The top of stack (the caller save buffer) now looks like this:
-    //     sp[0 * ptrSize]: probeFunction
-    //     sp[1 * ptrSize]: arg1
-    //     sp[2 * ptrSize]: address of arm64ProbeTrampoline()
-    //     sp[3 * ptrSize]: saved x27
-    //     sp[4 * ptrSize]: saved x28
-    //     sp[5 * ptrSize]: saved lr
-    //     sp[6 * ptrSize]: saved sp
+    // MacroAssemblerARM64::probe() has already generated code to store some values in an
+    // IncomingProbeRecord. sp points to the IncomingProbeRecord.
 
+    "mov       x26, sp" "\n"
     "mov       x27, sp" "\n"
-    "mov       x28, sp" "\n"
-
-    "sub       x28, x28, #" STRINGIZE_VALUE_OF(PROBE_ALIGNED_SIZE) "\n"
-
-    // The ARM EABI specifies that the stack needs to be 16 byte aligned.
-    "bic       x28, x28, #0xf" "\n"
-    "mov       sp, x28" "\n"
-
-    "str       x27, [sp, #" STRINGIZE_VALUE_OF(SAVED_CALLER_SP) "]" "\n"
-
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n"
-    "str       x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X1_OFFSET) "]" "\n"
-    "str       x2, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X2_OFFSET) "]" "\n"
-    "str       x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X3_OFFSET) "]" "\n"
-    "str       x4, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X4_OFFSET) "]" "\n"
-    "str       x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X5_OFFSET) "]" "\n"
-    "str       x6, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X6_OFFSET) "]" "\n"
-    "str       x7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X7_OFFSET) "]" "\n"
-    "str       x8, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X8_OFFSET) "]" "\n"
-    "str       x9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X9_OFFSET) "]" "\n"
-    "str       x10, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X10_OFFSET) "]" "\n"
-    "str       x11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X11_OFFSET) "]" "\n"
-    "str       x12, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X12_OFFSET) "]" "\n"
-    "str       x13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X13_OFFSET) "]" "\n"
-    "str       x14, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X14_OFFSET) "]" "\n"
-    "str       x15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X15_OFFSET) "]" "\n"
-    "str       x16, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X16_OFFSET) "]" "\n"
-    "str       x17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X17_OFFSET) "]" "\n"
-    "str       x18, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X18_OFFSET) "]" "\n"
-    "str       x19, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X19_OFFSET) "]" "\n"
-    "str       x20, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X20_OFFSET) "]" "\n"
-    "str       x21, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X21_OFFSET) "]" "\n"
-    "str       x22, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X22_OFFSET) "]" "\n"
-    "str       x23, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X23_OFFSET) "]" "\n"
-    "str       x24, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X24_OFFSET) "]" "\n"
-    "str       x25, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X25_OFFSET) "]" "\n"
-    "str       x26, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X26_OFFSET) "]" "\n"
-
-    "ldr       x0, [x27, #3 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X27_OFFSET) "]" "\n"
-    "ldr       x0, [x27, #4 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X28_OFFSET) "]" "\n"
-
-    "str       fp, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FP_OFFSET) "]" "\n"
-
-    "ldr       x0, [x27, #5 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_LR_OFFSET) "]" "\n"
-    "ldr       x0, [x27, #6 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n"
 
+    "sub       x27, x27, #" STRINGIZE_VALUE_OF(PROBE_SIZE_PLUS_EXTRAS) "\n"
+    "bic       x27, x27, #0xf" "\n" // The ARM EABI specifies that the stack needs to be 16 byte aligned.
+    "mov       sp, x27" "\n" // Make sure interrupts don't over-write our data on the stack.
+
+    "stp       x0, x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n"
+    "mrs       x0, nzcv" "\n" // Preload nzcv.
+    "stp       x2, x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X2_OFFSET) "]" "\n"
+    "stp       x4, x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X4_OFFSET) "]" "\n"
+    "mrs       x1, fpsr" "\n" // Preload fpsr.
+    "stp       x6, x7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X6_OFFSET) "]" "\n"
+    "stp       x8, x9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X8_OFFSET) "]" "\n"
+
+    "ldp       x2, x3, [x26, #" STRINGIZE_VALUE_OF(IN_HANDLER_FUNCTION_OFFSET) "]" "\n" // Preload probe handler function and probe arg.
+    "ldp       x4, x5, [x26, #" STRINGIZE_VALUE_OF(IN_X26_OFFSET) "]" "\n" // Preload saved r26 and r27.
+    "ldp       x6, x7, [x26, #" STRINGIZE_VALUE_OF(IN_LR_OFFSET) "]" "\n" // Preload saved lr and sp.
+    "ldr       x8, [x26, #" STRINGIZE_VALUE_OF(IN_ERROR_FUNCTION_OFFSET) "]" "\n" // Preload probe error function.
+
+    "stp       x10, x11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X10_OFFSET) "]" "\n"
+    "stp       x12, x13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X12_OFFSET) "]" "\n"
+    "stp       x14, x15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X14_OFFSET) "]" "\n"
+    "stp       x16, x17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X16_OFFSET) "]" "\n"
+    "stp       x18, x19, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X18_OFFSET) "]" "\n"
+    "stp       x20, x21, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X20_OFFSET) "]" "\n"
+    "stp       x22, x23, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X22_OFFSET) "]" "\n"
+    "stp       x24, x25, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X24_OFFSET) "]" "\n"
+    "stp       x4, x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X26_OFFSET) "]" "\n" // Store saved r26 and r27 (preloaded into x4 and x5 above).
+    "stp       x28, fp, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X28_OFFSET) "]" "\n"
+    "stp       x6, x7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_LR_OFFSET) "]" "\n" // Save values lr and sp (preloaded into x6 and x7 above).
+
+    "str       x6, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_LR_OFFSET) "]" "\n" // Save a duplicate copy of lr (in x6).
+    "str       lr, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_RETURN_PC_OFFSET) "]" "\n" // Save a duplicate copy of return pc (in lr).
+
+    "add       lr, lr, #" STRINGIZE_VALUE_OF(2 * PTR_SIZE) "\n" // The PC after the probe is at 2 instructions past the return point.
     "str       lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n"
 
-    "mrs       x0, nzcv" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n"
-    "mrs       x0, fpsr" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FPSR_OFFSET) "]" "\n"
-
-    "ldr       x0, [x27, #0 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_PROBE_FUNCTION_OFFSET) "]" "\n"
-    "ldr       x0, [x27, #1 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
-    "str       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_ARG_OFFSET) "]" "\n"
-
-    "str       d0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q0_OFFSET) "]" "\n"
-    "str       d1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q1_OFFSET) "]" "\n"
-    "str       d2, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q2_OFFSET) "]" "\n"
-    "str       d3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q3_OFFSET) "]" "\n"
-    "str       d4, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q4_OFFSET) "]" "\n"
-    "str       d5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q5_OFFSET) "]" "\n"
-    "str       d6, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q6_OFFSET) "]" "\n"
-    "str       d7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q7_OFFSET) "]" "\n"
-    "str       d8, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q8_OFFSET) "]" "\n"
-    "str       d9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q9_OFFSET) "]" "\n"
-    "str       d10, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q10_OFFSET) "]" "\n"
-    "str       d11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q11_OFFSET) "]" "\n"
-    "str       d12, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q12_OFFSET) "]" "\n"
-    "str       d13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q13_OFFSET) "]" "\n"
-    "str       d14, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q14_OFFSET) "]" "\n"
-    "str       d15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q15_OFFSET) "]" "\n"
-    "str       d16, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q16_OFFSET) "]" "\n"
-    "str       d17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q17_OFFSET) "]" "\n"
-    "str       d18, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q18_OFFSET) "]" "\n"
-    "str       d19, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q19_OFFSET) "]" "\n"
-    "str       d20, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q20_OFFSET) "]" "\n"
-    "str       d21, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q21_OFFSET) "]" "\n"
-    "str       d22, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q22_OFFSET) "]" "\n"
-    "str       d23, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q23_OFFSET) "]" "\n"
-    "str       d24, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q24_OFFSET) "]" "\n"
-    "str       d25, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q25_OFFSET) "]" "\n"
-    "str       d26, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q26_OFFSET) "]" "\n"
-    "str       d27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q27_OFFSET) "]" "\n"
-    "str       d28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q28_OFFSET) "]" "\n"
-    "str       d29, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q29_OFFSET) "]" "\n"
-    "str       d30, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q30_OFFSET) "]" "\n"
-    "str       d31, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q31_OFFSET) "]" "\n"
-
-    "mov       x28, sp" "\n" // Save the ProbeContext*.
-
-    "mov       x0, sp" "\n" // the ProbeContext* arg.
-    "ldr       x27, [x27, #3 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
-    "blr       x27" "\n"
-
-    "mov       sp, x28" "\n"
+    "stp       x0, x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n" // Store nzcv and fpsr (preloaded into x0 and x1 above).
+
+    "stp       x2, x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_PROBE_FUNCTION_OFFSET) "]" "\n" // Store the probe handler function and arg (preloaded into x2 and x3 above).
+    "str       x8, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_ERROR_FUNCTION_OFFSET) "]" "\n" // Store the probe handler function and arg (preloaded into x8 above).
+
+    "add       x9, sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q0_OFFSET) "\n"
+    "stp       d0, d1, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q0)) "]" "\n"
+    "stp       d2, d3, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q2)) "]" "\n"
+    "stp       d4, d5, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q4)) "]" "\n"
+    "stp       d6, d7, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q6)) "]" "\n"
+    "stp       d8, d9, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q8)) "]" "\n"
+    "stp       d10, d11, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q10)) "]" "\n"
+    "stp       d12, d13, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q12)) "]" "\n"
+    "stp       d14, d15, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q14)) "]" "\n"
+    "stp       d16, d17, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q16)) "]" "\n"
+    "stp       d18, d19, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q18)) "]" "\n"
+    "stp       d20, d21, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q20)) "]" "\n"
+    "stp       d22, d23, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q22)) "]" "\n"
+    "stp       d24, d25, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q24)) "]" "\n"
+    "stp       d26, d27, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q26)) "]" "\n"
+    "stp       d28, d29, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q28)) "]" "\n"
+    "stp       d30, d31, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q30)) "]" "\n"
+
+    "mov       x27, sp" "\n" // Save the ProbeContext* in a callee saved register.
+
+    // Note: we haven't changed the value of fp. Hence, it is still pointing to the frame of
+    // the caller of the probe (which is what we want in order to play nice with debuggers e.g. lldb).
+    "mov       x0, sp" "\n" // Set the ProbeContext* arg.
+    "blr       x2" "\n" // Call the probe handler function (loaded into x2 above).
 
-    // To enable probes to modify register state, we copy all registers
-    // out of the ProbeContext before returning. That is except for x18, pc and sp.
+    "mov       sp, x27" "\n"
 
+    // To enable probes to modify register state, we copy all registers
+    // out of the ProbeContext before returning. That is except for x18.
     // x18 is "reserved for the platform. Conforming software should not make use of it."
     // Hence, the JITs would not be using it, and the probe should also not be modifying it.
     // See https://developer.apple.com/library/ios/documentation/Xcode/Conceptual/iPhoneOSABIReference/Articles/ARM64FunctionCallingConventions.html.
 
-    // We can't modify the pc, because the only way to set its value on ARM64 is via
-    // an indirect branch or a ret, which means we'll need a free register to do so.
-    // The probe mechanism is required to not perturb any registers that the caller
-    // may use. Hence, we don't have this free register available.
-
-    // In order to return to the caller, we need to ret via lr. The probe mechanism will
-    // restore lr's value after returning to the caller by loading the restore value
-    // from the caller save buffer. The caller expects to access the caller save buffer via
-    // sp. Hence, we cannot allow sp to be modified by the probe.
-
-    "ldr       d0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q0_OFFSET) "]" "\n"
-    "ldr       d1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q1_OFFSET) "]" "\n"
-    "ldr       d2, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q2_OFFSET) "]" "\n"
-    "ldr       d3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q3_OFFSET) "]" "\n"
-    "ldr       d4, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q4_OFFSET) "]" "\n"
-    "ldr       d5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q5_OFFSET) "]" "\n"
-    "ldr       d6, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q6_OFFSET) "]" "\n"
-    "ldr       d7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q7_OFFSET) "]" "\n"
-    "ldr       d8, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q8_OFFSET) "]" "\n"
-    "ldr       d9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q9_OFFSET) "]" "\n"
-    "ldr       d10, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q10_OFFSET) "]" "\n"
-    "ldr       d11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q11_OFFSET) "]" "\n"
-    "ldr       d12, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q12_OFFSET) "]" "\n"
-    "ldr       d13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q13_OFFSET) "]" "\n"
-    "ldr       d14, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q14_OFFSET) "]" "\n"
-    "ldr       d15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q15_OFFSET) "]" "\n"
-    "ldr       d16, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q16_OFFSET) "]" "\n"
-    "ldr       d17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q17_OFFSET) "]" "\n"
-    "ldr       d18, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q18_OFFSET) "]" "\n"
-    "ldr       d19, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q19_OFFSET) "]" "\n"
-    "ldr       d20, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q20_OFFSET) "]" "\n"
-    "ldr       d21, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q21_OFFSET) "]" "\n"
-    "ldr       d22, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q22_OFFSET) "]" "\n"
-    "ldr       d23, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q23_OFFSET) "]" "\n"
-    "ldr       d24, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q24_OFFSET) "]" "\n"
-    "ldr       d25, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q25_OFFSET) "]" "\n"
-    "ldr       d26, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q26_OFFSET) "]" "\n"
-    "ldr       d27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q27_OFFSET) "]" "\n"
-    "ldr       d28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q28_OFFSET) "]" "\n"
-    "ldr       d29, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q29_OFFSET) "]" "\n"
-    "ldr       d30, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q30_OFFSET) "]" "\n"
-    "ldr       d31, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q31_OFFSET) "]" "\n"
-
-    "ldr       x0, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n"
-    "ldr       x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X1_OFFSET) "]" "\n"
-    "ldr       x2, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X2_OFFSET) "]" "\n"
-    "ldr       x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X3_OFFSET) "]" "\n"
-    "ldr       x4, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X4_OFFSET) "]" "\n"
-    "ldr       x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X5_OFFSET) "]" "\n"
-    "ldr       x6, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X6_OFFSET) "]" "\n"
-    "ldr       x7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X7_OFFSET) "]" "\n"
-    "ldr       x8, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X8_OFFSET) "]" "\n"
-    "ldr       x9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X9_OFFSET) "]" "\n"
-    "ldr       x10, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X10_OFFSET) "]" "\n"
-    "ldr       x11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X11_OFFSET) "]" "\n"
-    "ldr       x12, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X12_OFFSET) "]" "\n"
-    "ldr       x13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X13_OFFSET) "]" "\n"
-    "ldr       x14, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X14_OFFSET) "]" "\n"
-    "ldr       x15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X15_OFFSET) "]" "\n"
-    "ldr       x16, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X16_OFFSET) "]" "\n"
-    "ldr       x17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X17_OFFSET) "]" "\n"
+    "add       x9, sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_Q0_OFFSET) "\n"
+    "ldp       d0, d1, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q0)) "]" "\n"
+    "ldp       d2, d3, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q2)) "]" "\n"
+    "ldp       d4, d5, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q4)) "]" "\n"
+    "ldp       d6, d7, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q6)) "]" "\n"
+    "ldp       d8, d9, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q8)) "]" "\n"
+    "ldp       d10, d11, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q10)) "]" "\n"
+    "ldp       d12, d13, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q12)) "]" "\n"
+    "ldp       d14, d15, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q14)) "]" "\n"
+    "ldp       d16, d17, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q16)) "]" "\n"
+    "ldp       d18, d19, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q18)) "]" "\n"
+    "ldp       d20, d21, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q20)) "]" "\n"
+    "ldp       d22, d23, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q22)) "]" "\n"
+    "ldp       d24, d25, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q24)) "]" "\n"
+    "ldp       d26, d27, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q26)) "]" "\n"
+    "ldp       d28, d29, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q28)) "]" "\n"
+    "ldp       d30, d31, [x9, #" STRINGIZE_VALUE_OF(FPR_OFFSET(Q30)) "]" "\n"
+
+    "ldp       x0, x1, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n"
+    "ldp       x2, x3, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X2_OFFSET) "]" "\n"
+    "ldp       x4, x5, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X4_OFFSET) "]" "\n"
+    "ldp       x6, x7, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X6_OFFSET) "]" "\n"
+    "ldp       x8, x9, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X8_OFFSET) "]" "\n"
+    "ldp       x10, x11, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X10_OFFSET) "]" "\n"
+    "ldp       x12, x13, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X12_OFFSET) "]" "\n"
+    "ldp       x14, x15, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X14_OFFSET) "]" "\n"
+    "ldp       x16, x17, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X16_OFFSET) "]" "\n"
     // x18 should not be modified by the probe. See comment above for details.
-    "ldr       x19, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X19_OFFSET) "]" "\n"
-    "ldr       x20, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X20_OFFSET) "]" "\n"
-    "ldr       x21, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X21_OFFSET) "]" "\n"
-    "ldr       x22, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X22_OFFSET) "]" "\n"
-    "ldr       x23, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X23_OFFSET) "]" "\n"
-    "ldr       x24, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X24_OFFSET) "]" "\n"
-    "ldr       x25, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X25_OFFSET) "]" "\n"
-    "ldr       x26, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X26_OFFSET) "]" "\n"
-
-    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FPSR_OFFSET) "]" "\n"
-    "msr       fpsr, x27" "\n"
-
-    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n"
-    "msr       nzcv, x27" "\n"
-    "ldr       fp, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FP_OFFSET) "]" "\n"
+    "ldp       x19, x20, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X19_OFFSET) "]" "\n"
+    "ldp       x21, x22, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X21_OFFSET) "]" "\n"
+    "ldp       x23, x24, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X23_OFFSET) "]" "\n"
+    "ldp       x25, x26, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X25_OFFSET) "]" "\n"
+
+    // Remaining registers to restore are: fpsr, nzcv, x27, x28, fp, lr, sp, and pc.
+
+    "mov       lr, #" STRINGIZE_VALUE_OF(STATE_PC_NOT_CHANGED) "\n"
+
+    // The only way to set the pc on ARM64 (from user space) is via an indirect branch
+    // or a ret, which means we'll need a free register to do so. For our purposes, lr
+    // happens to be available in applications of the probe where we may want to
+    // continue executing at a different location (i.e. change the pc) after the probe
+    // returns. So, the ARM64 probe implementation will allow the probe handler to
+    // either modify lr or pc, but not both in the same probe invocation. The probe
+    // mechanism ensures that we never try to modify both lr and pc, else it will
+    // fail with a RELEASE_ASSERT_NOT_REACHED in arm64ProbeError().
+
+    // Determine if the probe handler changed the pc.
+    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_RETURN_PC_OFFSET) "]" "\n"
+    "ldr       x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n"
+    "add       x27, x27, #" STRINGIZE_VALUE_OF(2 * PTR_SIZE) "\n"
+    "cmp       x27, x28" "\n"
+    "beq     " LOCAL_LABEL_STRING(ctiMasmProbeTrampolinePrepareOutgoingRecords) "\n"
+
+    // pc was changed. Determine if the probe handler also changed lr.
+    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_LR_OFFSET) "]" "\n"
+    "ldr       x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_LR_OFFSET) "]" "\n"
+    "cmp       x27, x28" "\n"
+    "bne     " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineError) "\n"
+
+    "mov       lr, #" STRINGIZE_VALUE_OF(STATE_PC_CHANGED) "\n"
+
+    LOCAL_LABEL_STRING(ctiMasmProbeTrampolinePrepareOutgoingRecords) ":" "\n"
+
+    "ldr       fp, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_RETURN_PC_OFFSET) "]" "\n" // Preload the probe return site pc.
+
+    // The probe handler may have moved the sp. For the return process, we may need
+    // space for 2 OutgoingProbeRecords below the final sp value. We need to make
+    // sure that the space for these 2 OutgoingProbeRecords do not overlap the
+    // restore values of the registers.
+
+    // All restore values are located at offset <= PROBE_CPU_FPSR_OFFSET. Hence,
+    // we need to make sure that resultant sp > offset of fpsr + 2 * sizeof(OutgoingProbeRecord).
 
-    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X27_OFFSET) "]" "\n"
-    "ldr       x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X28_OFFSET) "]" "\n"
+    "add       x27, sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FPSR_OFFSET + 2 * OUT_SIZE) "\n"
+    "ldr       x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n"
+    "cmp       x28, x27" "\n"
+    "bgt     " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineFillOutgoingProbeRecords) "\n"
 
-    // There are 4 more registers left to restore: x27, x28, lr, sp, and pc.
-    // The JIT code's lr and sp will be restored by the caller.
+    // There is overlap. We need to copy the ProbeContext to a safe area first.
+    // Let's locate the "safe area" at 2x sizeof(ProbeContext) below where the OutgoingProbeRecords are.
+    // This ensures that:
+    // 1. The safe area does not overlap the OutgoingProbeRecords.
+    // 2. The safe area does not overlap the ProbeContext.
 
-    // Restore pc by loading it into lr. The ret below will put in the pc.
-    "ldr       lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n"
+    // x28 already contains [sp, #STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET)].
+    "sub       x28, x28, #" STRINGIZE_VALUE_OF(2 * PROBE_SIZE) "\n"
 
-    // We need x27 as a scratch register to help with popping the ProbeContext.
-    // Hence, before we pop the ProbeContext, we need to copy the restore value
-    // for x27 from the ProbeContext to the caller save buffer.
-    "ldr       x28, [sp, #" STRINGIZE_VALUE_OF(SAVED_CALLER_SP) "]" "\n"
-    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X27_OFFSET) "]" "\n"
-    "str       x27, [x28, #4 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
+    "mov       x27, sp" "\n" // Save the original ProbeContext*.
+
+    // Make sure the stack pointer points to the safe area. This ensures that the
+    // safe area is protected from interrupt handlers overwriting it.
+    "mov       sp, x28" "\n" // sp now points to the new ProbeContext in the safe area.
+
+    // Copy the relevant restore data to the new ProbeContext*.
+    "str       lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n" // Stash the pc changed state away so that we can use lr.
+
+    "ldp       x28, lr, [x27, #" STRINGIZE_VALUE_OF(PROBE_CPU_X27_OFFSET) "]" "\n" // copy x27 and x28.
+    "stp       x28, lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X27_OFFSET) "]" "\n"
+    "ldp       x28, lr, [x27, #" STRINGIZE_VALUE_OF(PROBE_CPU_FP_OFFSET) "]" "\n" // copy fp and lr.
+    "stp       x28, lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FP_OFFSET) "]" "\n"
+    "ldp       x28, lr, [x27, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n" // copy sp and pc.
+    "stp       x28, lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n"
+    "ldp       x28, lr, [x27, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n" // copy nzcv and fpsr.
+    "stp       x28, lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n"
+
+    "ldr       lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X0_OFFSET) "]" "\n" // Retrieve the stashed the pc changed state.
+
+    LOCAL_LABEL_STRING(ctiMasmProbeTrampolineFillOutgoingProbeRecords) ":" "\n"
+
+    "cbnz       lr, " LOCAL_LABEL_STRING(ctiMasmProbeTrampolineEnd) "\n" // Skip lr restoration setup if state (in lr) == STATE_PC_CHANGED.
+
+    // In order to restore lr, we need to do the restoration at the probe return site.
+    // The probe return site expects sp to be pointing at an OutgoingProbeRecord such that
+    // popping the OutgoingProbeRecord will yield the desired sp. The probe return site
+    // also expects the lr value to be restored is stashed in the OutgoingProbeRecord.
+    // We can make this happen by pushing 2 OutgoingProbeRecords instead of 1:
+    // 1 for the probe return site, and 1 at ctiMasmProbeTrampolineEnd for returning from
+    // this probe.
+
+    // Fill in the OutgoingProbeStack for the probe return site.
+    "ldr       lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n"
+    "sub       lr, lr, #" STRINGIZE_VALUE_OF(OUT_SIZE) "\n"
 
-    // Since lr is also restored by the caller, we need to copy its restore
-    // value to the caller save buffer too.
     "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_LR_OFFSET) "]" "\n"
-    "str       x27, [x28, #6 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
+    "str       x27, [lr, #" STRINGIZE_VALUE_OF(OUT_LR_OFFSET) "]" "\n"
 
-    // We're now done with x28, and can restore its value.
-    "ldr       x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X28_OFFSET) "]" "\n"
+    // Set up the sp and pc values so that ctiMasmProbeTrampolineEnd will return to the probe return site.
+    "str       lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n"
+    "str       fp, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n" // Store the probe return site pc (preloaded into fp above).
 
-    // We're now done with the ProbeContext, and can pop it to restore sp so that
-    // it points to the caller save buffer.
-    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(SAVED_CALLER_SP) "]" "\n"
-    "mov       sp, x27" "\n"
+    LOCAL_LABEL_STRING(ctiMasmProbeTrampolineEnd) ":" "\n"
 
-    // We're now done with x27, and can restore it.
-    "ldr       x27, [sp, #4 * " STRINGIZE_VALUE_OF(PTR_SIZE) "]" "\n"
+    // Fill in the OutgoingProbeStack.
+    "ldr       lr, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_SP_OFFSET) "]" "\n"
+    "sub       lr, lr, #" STRINGIZE_VALUE_OF(OUT_SIZE) "\n"
 
+    "ldp       x27, x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_NZCV_OFFSET) "]" "\n"
+    "stp       x27, x28, [lr, #" STRINGIZE_VALUE_OF(OUT_NZCV_OFFSET) "]" "\n"
+    "ldp       x27, x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_X27_OFFSET) "]" "\n"
+    "stp       x27, x28, [lr, #" STRINGIZE_VALUE_OF(OUT_X27_OFFSET) "]" "\n"
+    "ldr       x27, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_FP_OFFSET) "]" "\n"
+    "ldr       x28, [sp, #" STRINGIZE_VALUE_OF(PROBE_CPU_PC_OFFSET) "]" "\n"
+    "stp       x27, x28, [lr, #" STRINGIZE_VALUE_OF(OUT_FP_OFFSET) "]" "\n"
+    "mov       sp, lr" "\n"
+
+    // Restore the remaining registers and pop the OutgoingProbeStack.
+    "ldp       x27, x28, [sp], #" STRINGIZE_VALUE_OF(2 * PTR_SIZE) "\n"
+    "msr       nzcv, x27" "\n"
+    "msr       fpsr, x28" "\n"
+    "ldp       x27, x28, [sp], #" STRINGIZE_VALUE_OF(2 * PTR_SIZE) "\n"
+    "ldp       fp, lr, [sp], #" STRINGIZE_VALUE_OF(2 * PTR_SIZE) "\n"
     "ret" "\n"
+
+    LOCAL_LABEL_STRING(ctiMasmProbeTrampolineError) ":" "\n"
+    // The probe handler changed both lr and pc. This is not supported for ARM64.
+    "ldr       x1, [sp, #" STRINGIZE_VALUE_OF(SAVED_PROBE_ERROR_FUNCTION_OFFSET) "]" "\n"
+    "mov       x0, sp" "\n" // Set the ProbeContext* arg.
+    "blr       x1" "\n"
+    "brk       #0x1000" // Should never return here.
 );
 #endif // COMPILER(GCC_OR_CLANG)
 
-static void arm64ProbeTrampoline(ProbeContext* context)
+static NO_RETURN_DUE_TO_CRASH void arm64ProbeError(ProbeContext*)
 {
-    void* origSP = context->cpu.sp();
-    void* origPC = context->cpu.pc();
-    
-    context->probeFunction(context);
-    
-    if (context->cpu.sp() != origSP) {
-        dataLog("MacroAssembler probe ERROR: ARM64 does not support the probe changing the SP. The change will be ignored\n");
-        context->cpu.sp() = origSP;
-    }
-    if (context->cpu.pc() != origPC) {
-        dataLog("MacroAssembler probe ERROR: ARM64 does not support the probe changing the PC. The change will be ignored\n");
-        context->cpu.pc() = origPC;
-    }
+    dataLog("MacroAssembler probe ERROR: ARM64 does not support the probe changing both LR and PC.\n");
+    RELEASE_ASSERT_NOT_REACHED();
 }
 
 void MacroAssembler::probe(ProbeFunction function, void* arg)
 {
-    sub64(TrustedImm32(7 * 8), sp);
-
-    store64(x27, Address(sp, 3 * 8));
-    store64(x28, Address(sp, 4 * 8));
-    store64(lr, Address(sp, 5 * 8));
-
-    add64(TrustedImm32(7 * 8), sp, x28);
-    store64(x28, Address(sp, 6 * 8)); // Save original sp value.
-
-    move(TrustedImmPtr(reinterpret_cast<void*>(function)), x28);
-    store64(x28, Address(sp));
-    move(TrustedImmPtr(arg), x28);
-    store64(x28, Address(sp, 1 * 8));
-    move(TrustedImmPtr(reinterpret_cast<void*>(arm64ProbeTrampoline)), x28);
-    store64(x28, Address(sp, 2 * 8));
-
-    move(TrustedImmPtr(reinterpret_cast<void*>(ctiMasmProbeTrampoline)), x28);
-    m_assembler.blr(x28);
-
-    // ctiMasmProbeTrampoline should have restored every register except for
-    // lr and the sp.
-    load64(Address(sp, 5 * 8), lr);
-    add64(TrustedImm32(7 * 8), sp);
+    sub64(TrustedImm32(sizeof(IncomingProbeRecord)), sp);
+
+    storePair64(x26, x27, sp, TrustedImm32(offsetof(IncomingProbeRecord, x26)));
+    add64(TrustedImm32(sizeof(IncomingProbeRecord)), sp, x26);
+    storePair64(lr, x26, sp, TrustedImm32(offsetof(IncomingProbeRecord, lr))); // Save lr and original sp value.
+    move(TrustedImmPtr(reinterpret_cast<void*>(function)), x26);
+    move(TrustedImmPtr(arg), x27);
+    storePair64(x26, x27, sp, TrustedImm32(offsetof(IncomingProbeRecord, probeHandlerFunction)));
+    move(TrustedImmPtr(reinterpret_cast<void*>(arm64ProbeError)), x27);
+    store64(x27, Address(sp, offsetof(IncomingProbeRecord, probeErrorFunction)));
+
+    move(TrustedImmPtr(reinterpret_cast<void*>(ctiMasmProbeTrampoline)), x26);
+    m_assembler.blr(x26);
+
+    // ctiMasmProbeTrampoline should have restored every register except for lr and the sp.
+    load64(Address(sp, offsetof(OutgoingProbeRecord, lr)), lr);
+    add64(TrustedImm32(sizeof(OutgoingProbeRecord)), sp);
 }
 #endif // ENABLE(MASM_PROBE)
 

diff --git a/Source/JavaScriptCore/assembler/testmasm.cpp b/Source/JavaScriptCore/assembler/testmasm.cpp
index d0e6d4f..a3fdd81 100644 (file)
--- a/Source/JavaScriptCore/assembler/testmasm.cpp
+++ b/Source/JavaScriptCore/assembler/testmasm.cpp
@@ -71,6 +71,7 @@ template<typename T> T nextID(T id) { return static_cast<T>(id + 1); }
 #else
 #define testWord(x) testWord32(x)
 #endif
+#define testDoubleWord(x) static_cast<double>(testWord(x))
 
 // Nothing fancy for now; we just use the existing WTF assertion machinery.
 #define CHECK(x) do {                                                   \
@@ -81,6 +82,9 @@ template<typename T> T nextID(T id) { return static_cast<T>(id + 1); }
         CRASH();                                                        \
     } while (false)
 
+#define CHECK_DOUBLE_BITWISE_EQ(a, b) \
+    CHECK(bitwise_cast<uint64_t>(a) == bitwise_cast<uint64_t>(a))
+
 #if ENABLE(MASM_PROBE)
 bool isPC(MacroAssembler::RegisterID id)
 {
@@ -101,6 +105,17 @@ bool isFP(MacroAssembler::RegisterID id)
 {
     return id == MacroAssembler::framePointerRegister;
 }
+
+bool isSpecialGPR(MacroAssembler::RegisterID id)
+{
+    if (isPC(id) || isSP(id) || isFP(id))
+        return true;
+#if CPU(ARM64)
+    if (id == ARM64Registers::x18)
+        return true;
+#endif
+    return false;
+}
 #endif // ENABLE(MASM_PROBE)
 
 MacroAssemblerCodeRef compile(Generator&& generate)
@@ -164,8 +179,8 @@ void testProbeReadsArgumentRegisters()
             CHECK(context->gpr(GPRInfo::argumentGPR2) == testWord(2));
             CHECK(context->gpr(GPRInfo::argumentGPR3) == testWord(3));
 
-            CHECK(context->fpr(FPRInfo::fpRegT0) == testWord32(0));
-            CHECK(context->fpr(FPRInfo::fpRegT1) == testWord32(1));
+            CHECK_DOUBLE_BITWISE_EQ(context->fpr(FPRInfo::fpRegT0), static_cast<double>(testWord32(0)));
+            CHECK_DOUBLE_BITWISE_EQ(context->fpr(FPRInfo::fpRegT1),  static_cast<double>(testWord32(1)));
         });
         jit.emitFunctionEpilogue();
         jit.ret();
@@ -217,8 +232,8 @@ void testProbeWritesArgumentRegisters()
             CHECK(context->gpr(GPRInfo::argumentGPR2) == testWord(2));
             CHECK(context->gpr(GPRInfo::argumentGPR3) == testWord(3));
 
-            CHECK(context->fpr(FPRInfo::fpRegT0) == testWord32(0));
-            CHECK(context->fpr(FPRInfo::fpRegT1) == testWord32(1));
+            CHECK_DOUBLE_BITWISE_EQ(context->fpr(FPRInfo::fpRegT0), static_cast<double>(testWord32(0)));
+            CHECK_DOUBLE_BITWISE_EQ(context->fpr(FPRInfo::fpRegT1), static_cast<double>(testWord32(1)));
         });
 
         jit.emitFunctionEpilogue();
@@ -256,13 +271,13 @@ void testProbePreservesGPRS()
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
                 originalState.gpr(id) = context->gpr(id);
-                if (isPC(id) || isSP(id) || isFP(id))
+                if (isSpecialGPR(id))
                     continue;
                 context->gpr(id) = testWord(static_cast<int>(id));
             }
             for (auto id = CCallHelpers::firstFPRegister(); id <= CCallHelpers::lastFPRegister(); id = nextID(id)) {
                 originalState.fpr(id) = context->fpr(id);
-                context->fpr(id) = testWord(id);
+                context->fpr(id) = testDoubleWord(id);
             }
         });
 
@@ -277,23 +292,23 @@ void testProbePreservesGPRS()
         jit.probe([&] (ProbeContext* context) {
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
-                if (isPC(id))
-                    continue;
                 if (isSP(id) || isFP(id)) {
                     CHECK(context->gpr(id) == originalState.gpr(id));
                     continue;
                 }
+                if (isSpecialGPR(id))
+                    continue;
                 CHECK(context->gpr(id) == testWord(id));
             }
             for (auto id = CCallHelpers::firstFPRegister(); id <= CCallHelpers::lastFPRegister(); id = nextID(id))
-                CHECK(context->fpr(id) == testWord(id));
+                CHECK_DOUBLE_BITWISE_EQ(context->fpr(id), testDoubleWord(id));
         });
 
         // Restore the original state.
         jit.probe([&] (ProbeContext* context) {
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
-                if (isPC(id) || isSP(id) || isFP(id))
+                if (isSpecialGPR(id))
                     continue;
                 context->gpr(id) = originalState.gpr(id);
             }
@@ -305,12 +320,12 @@ void testProbePreservesGPRS()
         jit.probe([&] (ProbeContext* context) {
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
-                if (isPC(id) || isSP(id) || isFP(id))
+                if (isSpecialGPR(id))
                     continue;
                 CHECK(context->gpr(id) == originalState.gpr(id));
             }
             for (auto id = CCallHelpers::firstFPRegister(); id <= CCallHelpers::lastFPRegister(); id = nextID(id))
-                CHECK(context->fpr(id) == originalState.fpr(id));
+                CHECK_DOUBLE_BITWISE_EQ(context->fpr(id), originalState.fpr(id));
         });
 
         jit.emitFunctionEpilogue();
@@ -325,8 +340,17 @@ void testProbeModifiesStackPointer(WTF::Function<void*(ProbeContext*)> computeMo
     MacroAssembler::CPUState originalState;
     uint8_t* originalSP { nullptr };
     void* modifiedSP { nullptr };
-#if CPU(X86) || CPU(X86_64) || CPU(ARM_THUMB2) || CPU(ARM_TRADITIONAL)
     uintptr_t modifiedFlags { 0 };
+    
+#if CPU(X86) || CPU(X86_64)
+    auto flagsSPR = X86Registers::eflags;
+    uintptr_t flagsMask = 0xc5;
+#elif CPU(ARM_THUMB2) || CPU(ARM_TRADITIONAL)
+    auto flagsSPR = ARMRegisters::apsr;
+    uintptr_t flagsMask = 0xf0000000;
+#elif CPU(ARM64)
+    auto flagsSPR = ARM64Registers::nzcv;
+    uintptr_t flagsMask = 0xf0000000;
 #endif
 
     compileAndRun<void>([&] (CCallHelpers& jit) {
@@ -338,7 +362,7 @@ void testProbeModifiesStackPointer(WTF::Function<void*(ProbeContext*)> computeMo
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
                 originalState.gpr(id) = context->gpr(id);
-                if (isPC(id) || isSP(id) || isFP(id))
+                if (isSpecialGPR(id))
                     continue;
                 context->gpr(id) = testWord(static_cast<int>(id));
             }
@@ -346,15 +370,11 @@ void testProbeModifiesStackPointer(WTF::Function<void*(ProbeContext*)> computeMo
                 originalState.fpr(id) = context->fpr(id);
                 context->fpr(id) = testWord(id);
             }
-#if CPU(X86) || CPU(X86_64)
-            originalState.spr(X86Registers::eflags) = context->spr(X86Registers::eflags);
-            modifiedFlags = originalState.spr(X86Registers::eflags) ^ 0xc5;
-            context->spr(X86Registers::eflags) = modifiedFlags;
-#elif CPU(ARM_THUMB2) || CPU(ARM_TRADITIONAL)
-            originalState.spr(ARMRegisters::apsr) = context->spr(ARMRegisters::apsr);
-            modifiedFlags = originalState.spr(ARMRegisters::apsr) ^ 0xf0000000;
-            context->spr(ARMRegisters::apsr) = modifiedFlags;
-#endif
+
+            originalState.spr(flagsSPR) = context->spr(flagsSPR);
+            modifiedFlags = originalState.spr(flagsSPR) ^ flagsMask;
+            context->spr(flagsSPR) = modifiedFlags;
+
             originalSP = reinterpret_cast<uint8_t*>(context->sp());
             modifiedSP = computeModifiedStack(context);
             context->sp() = modifiedSP;
@@ -364,21 +384,17 @@ void testProbeModifiesStackPointer(WTF::Function<void*(ProbeContext*)> computeMo
         jit.probe([&] (ProbeContext* context) {
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
-                if (isPC(id) || isSP(id))
-                    continue;
                 if (isFP(id)) {
                     CHECK(context->gpr(id) == originalState.gpr(id));
                     continue;
                 }
+                if (isSpecialGPR(id))
+                    continue;
                 CHECK(context->gpr(id) == testWord(id));
             }
             for (auto id = CCallHelpers::firstFPRegister(); id <= CCallHelpers::lastFPRegister(); id = nextID(id))
-                CHECK(context->fpr(id) == testWord(id));
-#if CPU(X86) || CPU(X86_64)
-            CHECK(context->spr(X86Registers::eflags) == modifiedFlags);
-#elif CPU(ARM_THUMB2) || CPU(ARM_TRADITIONAL)
-            CHECK(context->spr(ARMRegisters::apsr) == modifiedFlags);
-#endif
+                CHECK_DOUBLE_BITWISE_EQ(context->fpr(id), testDoubleWord(id));
+            CHECK(context->spr(flagsSPR) == modifiedFlags);
             CHECK(context->sp() == modifiedSP);
         });
 
@@ -386,17 +402,13 @@ void testProbeModifiesStackPointer(WTF::Function<void*(ProbeContext*)> computeMo
         jit.probe([&] (ProbeContext* context) {
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
-                if (isPC(id) || isSP(id) || isFP(id))
+                if (isSpecialGPR(id))
                     continue;
                 context->gpr(id) = originalState.gpr(id);
             }
             for (auto id = CCallHelpers::firstFPRegister(); id <= CCallHelpers::lastFPRegister(); id = nextID(id))
                 context->fpr(id) = originalState.fpr(id);
-#if CPU(X86) || CPU(X86_64)
-            context->spr(X86Registers::eflags) = originalState.spr(X86Registers::eflags);
-#elif CPU(ARM_THUMB2) || CPU(ARM_TRADITIONAL)
-            context->spr(ARMRegisters::apsr) = originalState.spr(ARMRegisters::apsr);
-#endif
+            context->spr(flagsSPR) = originalState.spr(flagsSPR);
             context->sp() = originalSP;
         });
 
@@ -404,17 +416,13 @@ void testProbeModifiesStackPointer(WTF::Function<void*(ProbeContext*)> computeMo
         jit.probe([&] (ProbeContext* context) {
             probeCallCount++;
             for (auto id = CCallHelpers::firstRegister(); id <= CCallHelpers::lastRegister(); id = nextID(id)) {
-                if (isPC(id) || isSP(id) || isFP(id))
+                if (isSpecialGPR(id))
                     continue;
                 CHECK(context->gpr(id) == originalState.gpr(id));
             }
             for (auto id = CCallHelpers::firstFPRegister(); id <= CCallHelpers::lastFPRegister(); id = nextID(id))
-                CHECK(context->fpr(id) == originalState.fpr(id));
-#if CPU(X86) || CPU(X86_64)
-            CHECK(context->spr(X86Registers::eflags) == originalState.spr(X86Registers::eflags));
-#elif CPU(ARM_THUMB2) || CPU(ARM_TRADITIONAL)
-            CHECK(context->spr(ARMRegisters::apsr) == originalState.spr(ARMRegisters::apsr));
-#endif
+                CHECK_DOUBLE_BITWISE_EQ(context->fpr(id),  originalState.fpr(id));
+            CHECK(context->spr(flagsSPR) == originalState.spr(flagsSPR));
             CHECK(context->sp() == originalSP);
         });
 
@@ -426,7 +434,12 @@ void testProbeModifiesStackPointer(WTF::Function<void*(ProbeContext*)> computeMo
 
 void testProbeModifiesStackPointerToInsideProbeContextOnStack()
 {
-    for (size_t offset = 0; offset < sizeof(ProbeContext); offset += sizeof(uintptr_t)) {
+    size_t increment = sizeof(uintptr_t);
+#if CPU(ARM64)
+    // The ARM64 probe uses ldp and stp which require 16 byte alignment.
+    increment = 2 * sizeof(uintptr_t);
+#endif
+    for (size_t offset = 0; offset < sizeof(ProbeContext); offset += increment) {
         testProbeModifiesStackPointer([=] (ProbeContext* context) -> void* {
             return reinterpret_cast<uint8_t*>(context) + offset;
         });
@@ -435,7 +448,12 @@ void testProbeModifiesStackPointerToInsideProbeContextOnStack()
 
 void testProbeModifiesStackPointerToNBytesBelowSP()
 {
-    for (size_t offset = 0; offset < 1 * KB; offset += sizeof(uintptr_t)) {
+    size_t increment = sizeof(uintptr_t);
+#if CPU(ARM64)
+    // The ARM64 probe uses ldp and stp which require 16 byte alignment.
+    increment = 2 * sizeof(uintptr_t);
+#endif
+    for (size_t offset = 0; offset < 1 * KB; offset += increment) {
         testProbeModifiesStackPointer([=] (ProbeContext* context) -> void* {
             return reinterpret_cast<uint8_t*>(context->cpu.sp()) - offset;
         });