Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp

   1 /*
   2  * Copyright (C) 2016 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "WasmB3IRGenerator.h"
  28
  29 #if ENABLE(WEBASSEMBLY)
  30
  31 #include "B3BasicBlockInlines.h"
  32 #include "B3CCallValue.h"
  33 #include "B3Compile.h"
  34 #include "B3ConstPtrValue.h"
  35 #include "B3FixSSA.h"
  36 #include "B3Generate.h"
  37 #include "B3StackmapGenerationParams.h"
  38 #include "B3SwitchValue.h"
  39 #include "B3Validate.h"
  40 #include "B3ValueInlines.h"
  41 #include "B3Variable.h"
  42 #include "B3VariableValue.h"
  43 #include "B3WasmAddressValue.h"
  44 #include "B3WasmBoundsCheckValue.h"
  45 #include "JSCInlines.h"
  46 #include "JSWebAssemblyInstance.h"
  47 #include "JSWebAssemblyModule.h"
  48 #include "JSWebAssemblyRuntimeError.h"
  49 #include "VirtualRegister.h"
  50 #include "WasmCallingConvention.h"
  51 #include "WasmExceptionType.h"
  52 #include "WasmFunctionParser.h"
  53 #include "WasmMemory.h"
  54 #include <wtf/Optional.h>
  55
  56 void dumpProcedure(void* ptr)
  57 {
  58     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
  59     proc->dump(WTF::dataFile());
  60 }
  61
  62 namespace JSC { namespace Wasm {
  63
  64 using namespace B3;
  65
  66 namespace {
  67 const bool verbose = false;
  68 }
  69
  70 class B3IRGenerator {
  71 public:
  72     struct ControlData {
  73         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
  74             : blockType(type)
  75             , continuation(continuation)
  76             , special(special)
  77         {
  78             if (signature != Void)
  79                 result.append(proc.addVariable(toB3Type(signature)));
  80         }
  81
  82         ControlData()
  83         {
  84         }
  85
  86         void dump(PrintStream& out) const
  87         {
  88             switch (type()) {
  89             case BlockType::If:
  90                 out.print("If:    ");
  91                 break;
  92             case BlockType::Block:
  93                 out.print("Block: ");
  94                 break;
  95             case BlockType::Loop:
  96                 out.print("Loop:  ");
  97                 break;
  98             }
  99             out.print("Continuation: ", *continuation, ", Special: ");
 100             if (special)
 101                 out.print(*special);
 102             else
 103                 out.print("None");
 104         }
 105
 106         BlockType type() const { return blockType; }
 107
 108         bool hasNonVoidSignature() const { return result.size(); }
 109
 110         BasicBlock* targetBlockForBranch()
 111         {
 112             if (type() == BlockType::Loop)
 113                 return special;
 114             return continuation;
 115         }
 116
 117         void convertIfToBlock()
 118         {
 119             ASSERT(type() == BlockType::If);
 120             blockType = BlockType::Block;
 121             special = nullptr;
 122         }
 123
 124     private:
 125         friend class B3IRGenerator;
 126         BlockType blockType;
 127         BasicBlock* continuation;
 128         BasicBlock* special;
 129         Vector<Variable*, 1> result;
 130     };
 131
 132     typedef Value* ExpressionType;
 133     typedef ControlData ControlType;
 134     typedef Vector<ExpressionType, 1> ExpressionList;
 135     typedef Vector<Variable*, 1> ResultList;
 136     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
 137
 138     static constexpr ExpressionType emptyExpression = nullptr;
 139
 140     typedef String ErrorType;
 141     typedef UnexpectedType<ErrorType> UnexpectedResult;
 142     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
 143     typedef Expected<void, ErrorType> PartialResult;
 144     template <typename ...Args>
 145     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
 146     {
 147         using namespace FailureHelper; // See ADL comment in WasmParser.h.
 148         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
 149     }
 150 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
 151         if (UNLIKELY(condition))                  \
 152             return fail(__VA_ARGS__);             \
 153     } while (0)
 154
 155     B3IRGenerator(VM&, const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, const ImmutableFunctionIndexSpace&);
 156
 157     PartialResult WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
 158     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
 159     ExpressionType addConstant(Type, uint64_t);
 160
 161     // Locals
 162     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
 163     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
 164
 165     // Globals
 166     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
 167     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
 168
 169     // Memory
 170     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
 171     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
 172
 173     // Basic operators
 174     template<OpType>
 175     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
 176     template<OpType>
 177     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
 178     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
 179
 180     // Control flow
 181     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
 182     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
 183     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
 184     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
 185     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
 186
 187     PartialResult WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
 188     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
 189     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
 190     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
 191     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
 192
 193     // Calls
 194     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 195     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 196     PartialResult WARN_UNUSED_RETURN addUnreachable();
 197
 198     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
 199
 200     void emitExceptionCheck(CCallHelpers&, ExceptionType);
 201
 202 private:
 203     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
 204     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
 205     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
 206
 207     void unify(Variable* target, const ExpressionType source);
 208     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
 209     Value* zeroForType(Type);
 210
 211     VM& m_vm;
 212     const ImmutableFunctionIndexSpace& m_functionIndexSpace;
 213     const ModuleInformation& m_info;
 214     Procedure& m_proc;
 215     BasicBlock* m_currentBlock;
 216     Vector<Variable*> m_locals;
 217     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
 218     GPRReg m_memoryBaseGPR;
 219     GPRReg m_memorySizeGPR;
 220     Value* m_zeroValues[numTypes];
 221     Value* m_instanceValue;
 222 };
 223
 224 B3IRGenerator::B3IRGenerator(VM& vm, const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace)
 225     : m_vm(vm)
 226     , m_functionIndexSpace(functionIndexSpace)
 227     , m_info(info)
 228     , m_proc(procedure)
 229     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
 230 {
 231     m_currentBlock = m_proc.addBlock();
 232
 233     for (unsigned i = 0; i < numTypes; ++i) {
 234         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
 235         case B3::Int32:
 236         case B3::Int64:
 237         case B3::Float:
 238         case B3::Double:
 239             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
 240             break;
 241         case B3::Void:
 242             m_zeroValues[i] = nullptr;
 243             break;
 244         }
 245     }
 246
 247     if (!!info.memory) {
 248         m_memoryBaseGPR = info.memory.pinnedRegisters().baseMemoryPointer;
 249         m_proc.pinRegister(m_memoryBaseGPR);
 250         ASSERT(!info.memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
 251         m_memorySizeGPR = info.memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
 252         for (const PinnedSizeRegisterInfo& regInfo : info.memory.pinnedRegisters().sizeRegisters)
 253             m_proc.pinRegister(regInfo.sizeRegister);
 254
 255         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
 256             AllowMacroScratchRegisterUsage allowScratch(jit);
 257             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
 258             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsMemoryAccess);
 259         });
 260
 261         B3::PatchpointValue* foo = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
 262         foo->setGenerator(
 263             [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 264                 AllowMacroScratchRegisterUsage allowScratch(jit);
 265             });
 266     }
 267
 268     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
 269
 270     m_instanceValue = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
 271         m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), &m_vm.topJSWebAssemblyInstance));
 272 }
 273
 274 void B3IRGenerator::emitExceptionCheck(CCallHelpers& jit, ExceptionType type)
 275 {
 276     jit.move(CCallHelpers::TrustedImm32(static_cast<uint32_t>(type)), GPRInfo::argumentGPR1);
 277     auto jumpToExceptionStub = jit.jump();
 278
 279     VM* vm = &m_vm;
 280     jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 281         linkBuffer.link(jumpToExceptionStub, CodeLocationLabel(vm->getCTIStub(throwExceptionFromWasmThunkGenerator).code()));
 282     });
 283 }
 284
 285 Value* B3IRGenerator::zeroForType(Type type)
 286 {
 287     ASSERT(type != Void);
 288     Value* zeroValue = m_zeroValues[linearizeType(type)];
 289     ASSERT(zeroValue);
 290     return zeroValue;
 291 }
 292
 293 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
 294 {
 295     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
 296
 297     for (uint32_t i = 0; i < count; ++i) {
 298         Variable* local = m_proc.addVariable(toB3Type(type));
 299         m_locals.uncheckedAppend(local);
 300         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
 301     }
 302     return { };
 303 }
 304
 305 auto B3IRGenerator::addArguments(const Vector<Type>& types) -> PartialResult
 306 {
 307     ASSERT(!m_locals.size());
 308     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(types.size()), "can't allocate memory for ", types.size(), " arguments");
 309
 310     m_locals.grow(types.size());
 311     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
 312         [&] (ExpressionType argument, unsigned i) {
 313             Variable* argumentVariable = m_proc.addVariable(argument->type());
 314             m_locals[i] = argumentVariable;
 315             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
 316         });
 317     return { };
 318 }
 319
 320 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
 321 {
 322     ASSERT(m_locals[index]);
 323     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
 324     return { };
 325 }
 326
 327 auto B3IRGenerator::addUnreachable() -> PartialResult
 328 {
 329     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
 330     unreachable->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 331         jit.breakpoint();
 332     });
 333     unreachable->effects.terminal = true;
 334     return { };
 335 }
 336
 337 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
 338 {
 339     ASSERT(m_locals[index]);
 340     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
 341     return { };
 342 }
 343
 344 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
 345 {
 346     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
 347     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), Origin(), globalsArray, index * sizeof(Register));
 348     return { };
 349 }
 350
 351 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
 352 {
 353     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
 354     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
 355     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, Origin(), value, globalsArray, index * sizeof(Register));
 356     return { };
 357 }
 358
 359 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
 360 {
 361     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
 362     ASSERT(sizeOfOperation + offset > offset);
 363     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
 364     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
 365     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
 366 }
 367
 368 inline uint32_t sizeOfLoadOp(LoadOpType op)
 369 {
 370     switch (op) {
 371     case LoadOpType::I32Load8S:
 372     case LoadOpType::I32Load8U:
 373     case LoadOpType::I64Load8S:
 374     case LoadOpType::I64Load8U:
 375         return 1;
 376     case LoadOpType::I32Load16S:
 377     case LoadOpType::I64Load16S:
 378     case LoadOpType::I32Load16U:
 379     case LoadOpType::I64Load16U:
 380         return 2;
 381     case LoadOpType::I32Load:
 382     case LoadOpType::I64Load32S:
 383     case LoadOpType::I64Load32U:
 384     case LoadOpType::F32Load:
 385         return 4;
 386     case LoadOpType::I64Load:
 387     case LoadOpType::F64Load:
 388         return 8;
 389     }
 390     RELEASE_ASSERT_NOT_REACHED();
 391 }
 392
 393 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
 394 {
 395     switch (op) {
 396     case LoadOpType::I32Load8S: {
 397         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 398     }
 399
 400     case LoadOpType::I64Load8S: {
 401         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 402         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 403     }
 404
 405     case LoadOpType::I32Load8U: {
 406         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 407     }
 408
 409     case LoadOpType::I64Load8U: {
 410         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 411         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 412     }
 413
 414     case LoadOpType::I32Load16S: {
 415         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 416     }
 417     case LoadOpType::I64Load16S: {
 418         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 419         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 420     }
 421
 422     case LoadOpType::I32Load: {
 423         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 424     }
 425
 426     case LoadOpType::I64Load32U: {
 427         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 428         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 429     }
 430
 431     case LoadOpType::I64Load32S: {
 432         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 433         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 434     }
 435
 436     case LoadOpType::I64Load: {
 437         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
 438     }
 439
 440     case LoadOpType::F32Load: {
 441         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
 442     }
 443
 444     case LoadOpType::F64Load: {
 445         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
 446     }
 447
 448     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
 449     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
 450     case LoadOpType::I32Load16U: {
 451         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 452         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
 453                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
 454     }
 455     case LoadOpType::I64Load16U: {
 456         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 457         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
 458                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
 459
 460         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), partialResult);
 461     }
 462     }
 463     RELEASE_ASSERT_NOT_REACHED();
 464 }
 465
 466 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
 467 {
 468     ASSERT(pointer->type() == Int32);
 469
 470     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
 471     return { };
 472 }
 473
 474 inline uint32_t sizeOfStoreOp(StoreOpType op)
 475 {
 476     switch (op) {
 477     case StoreOpType::I32Store8:
 478     case StoreOpType::I64Store8:
 479         return 1;
 480     case StoreOpType::I32Store16:
 481     case StoreOpType::I64Store16:
 482         return 2;
 483     case StoreOpType::I32Store:
 484     case StoreOpType::I64Store32:
 485     case StoreOpType::F32Store:
 486         return 4;
 487     case StoreOpType::I64Store:
 488     case StoreOpType::F64Store:
 489         return 8;
 490     }
 491     RELEASE_ASSERT_NOT_REACHED();
 492 }
 493
 494
 495 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
 496 {
 497     switch (op) {
 498     case StoreOpType::I64Store8:
 499         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 500         FALLTHROUGH;
 501
 502     case StoreOpType::I32Store8:
 503         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
 504         return;
 505
 506     case StoreOpType::I64Store16:
 507         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 508         FALLTHROUGH;
 509
 510     case StoreOpType::I32Store16:
 511         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
 512         return;
 513
 514     case StoreOpType::I64Store32:
 515         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 516         FALLTHROUGH;
 517
 518     case StoreOpType::I64Store:
 519     case StoreOpType::I32Store:
 520     case StoreOpType::F32Store:
 521     case StoreOpType::F64Store:
 522         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
 523         return;
 524     }
 525     RELEASE_ASSERT_NOT_REACHED();
 526 }
 527
 528 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
 529 {
 530     ASSERT(pointer->type() == Int32);
 531
 532     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
 533     return { };
 534 }
 535
 536 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
 537 {
 538     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
 539     return { };
 540 }
 541
 542 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
 543 {
 544     switch (type) {
 545     case Wasm::I32:
 546         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
 547     case Wasm::I64:
 548         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
 549     case Wasm::F32:
 550         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
 551     case Wasm::F64:
 552         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
 553     case Wasm::Void:
 554     case Wasm::Func:
 555     case Wasm::Anyfunc:
 556         break;
 557     }
 558     RELEASE_ASSERT_NOT_REACHED();
 559     return nullptr;
 560 }
 561
 562 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
 563 {
 564     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
 565 }
 566
 567 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
 568 {
 569     BasicBlock* body = m_proc.addBlock();
 570     BasicBlock* continuation = m_proc.addBlock();
 571     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
 572     body->addPredecessor(m_currentBlock);
 573     m_currentBlock = body;
 574     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
 575 }
 576
 577 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
 578 {
 579     // FIXME: This needs to do some kind of stack passing.
 580
 581     BasicBlock* taken = m_proc.addBlock();
 582     BasicBlock* notTaken = m_proc.addBlock();
 583     BasicBlock* continuation = m_proc.addBlock();
 584
 585     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 586     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
 587     taken->addPredecessor(m_currentBlock);
 588     notTaken->addPredecessor(m_currentBlock);
 589
 590     m_currentBlock = taken;
 591     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
 592     return { };
 593 }
 594
 595 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
 596 {
 597     unifyValuesWithBlock(currentStack, data.result);
 598     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 599     return addElseToUnreachable(data);
 600 }
 601
 602 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
 603 {
 604     ASSERT(data.type() == BlockType::If);
 605     m_currentBlock = data.special;
 606     data.convertIfToBlock();
 607     return { };
 608 }
 609
 610 auto B3IRGenerator::addReturn(const ExpressionList& returnValues) -> PartialResult
 611 {
 612     ASSERT(returnValues.size() <= 1);
 613     if (returnValues.size())
 614         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
 615     else
 616         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
 617     return { };
 618 }
 619
 620 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
 621 {
 622     if (data.type() != BlockType::Loop)
 623         unifyValuesWithBlock(returnValues, data.result);
 624
 625     BasicBlock* target = data.targetBlockForBranch();
 626     if (condition) {
 627         BasicBlock* continuation = m_proc.addBlock();
 628         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 629         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
 630         target->addPredecessor(m_currentBlock);
 631         continuation->addPredecessor(m_currentBlock);
 632         m_currentBlock = continuation;
 633     } else {
 634         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
 635         target->addPredecessor(m_currentBlock);
 636     }
 637
 638     return { };
 639 }
 640
 641 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
 642 {
 643     for (size_t i = 0; i < targets.size(); ++i)
 644         unifyValuesWithBlock(expressionStack, targets[i]->result);
 645     unifyValuesWithBlock(expressionStack, defaultTarget.result);
 646
 647     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
 648     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
 649     for (size_t i = 0; i < targets.size(); ++i)
 650         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
 651
 652     return { };
 653 }
 654
 655 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
 656 {
 657     ControlData& data = entry.controlData;
 658
 659     unifyValuesWithBlock(expressionStack, data.result);
 660     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 661     data.continuation->addPredecessor(m_currentBlock);
 662
 663     return addEndToUnreachable(entry);
 664 }
 665
 666
 667 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
 668 {
 669     ControlData& data = entry.controlData;
 670     m_currentBlock = data.continuation;
 671
 672     if (data.type() == BlockType::If) {
 673         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
 674         m_currentBlock->addPredecessor(data.special);
 675     }
 676
 677     for (Variable* result : data.result)
 678         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
 679
 680     return { };
 681 }
 682
 683 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
 684 {
 685     ASSERT(signature->arguments.size() == args.size());
 686
 687     Type returnType = signature->returnType;
 688
 689     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
 690         [&] (PatchpointValue* patchpoint) {
 691             patchpoint->effects.writesPinned = true;
 692             patchpoint->effects.readsPinned = true;
 693
 694             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 695                 AllowMacroScratchRegisterUsage allowScratch(jit);
 696
 697                 CCallHelpers::Call call = jit.call();
 698
 699                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 700                     m_unlinkedWasmToWasmCalls.append({ linkBuffer.locationOf(call), functionIndex });
 701                 });
 702             });
 703         });
 704     return { };
 705 }
 706
 707 auto B3IRGenerator::addCallIndirect(const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
 708 {
 709     ExpressionType calleeIndex = args.takeLast();
 710     ASSERT(signature->arguments.size() == args.size());
 711
 712     ExpressionType callableFunctionBuffer;
 713     ExpressionType callableFunctionBufferSize;
 714     {
 715         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
 716             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
 717         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
 718             table, JSWebAssemblyTable::offsetOfFunctions());
 719         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(),
 720             table, JSWebAssemblyTable::offsetOfSize());
 721     }
 722
 723     // Check the index we are looking for is valid.
 724     {
 725         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 726             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, Origin(), calleeIndex, callableFunctionBufferSize));
 727
 728         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 729             this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsCallIndirect);
 730         });
 731     }
 732
 733     // Compute the offset in the table index space we are looking for.
 734     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
 735         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
 736         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
 737     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), callableFunctionBuffer, offset);
 738
 739     // Check that the CallableFunction is initialized. We trap if it isn't. A null Signature* indicates it's not initialized.
 740     ExpressionType calleeSignature = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signature));
 741     {
 742         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 743             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(),
 744                 calleeSignature,
 745                 m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), 0)));
 746
 747         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 748             this->emitExceptionCheck(jit, ExceptionType::NullTableEntry);
 749         });
 750     }
 751
 752     // Check the signature matches the value we expect.
 753     {
 754         ExpressionType expectedSignature = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), signature);
 755         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 756             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignature, expectedSignature));
 757
 758         check->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 759             this->emitExceptionCheck(jit, ExceptionType::BadSignature);
 760         });
 761     }
 762
 763     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
 764
 765     Type returnType = signature->returnType;
 766     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
 767         [&] (PatchpointValue* patchpoint) {
 768             patchpoint->effects.writesPinned = true;
 769             patchpoint->effects.readsPinned = true;
 770
 771             patchpoint->append(calleeCode, ValueRep::SomeRegister);
 772
 773             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
 774                 jit.call(params[returnType == Void ? 0 : 1].gpr());
 775             });
 776         });
 777
 778     return { };
 779 }
 780
 781 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
 782 {
 783     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
 784 }
 785
 786 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
 787 {
 788     ASSERT(result.size() <= resultStack.size());
 789
 790     for (size_t i = 0; i < result.size(); ++i)
 791         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
 792 }
 793
 794 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
 795 {
 796     dataLogLn(comma, "ExpressionStack:");
 797     for (const auto& expression : expressionStack)
 798         dataLogLn(comma, *expression);
 799 }
 800
 801 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
 802 {
 803     dataLogLn("Processing Graph:");
 804     dataLog(m_proc);
 805     dataLogLn("With current block:", *m_currentBlock);
 806     dataLogLn("Control stack:");
 807     for (auto& data : controlStack) {
 808         dataLogLn("  ", data.controlData);
 809         if (data.enclosedExpressionStack.size()) {
 810             CommaPrinter comma("    ", "  with ");
 811             dumpExpressionStack(comma, data.enclosedExpressionStack);
 812         }
 813     }
 814
 815     CommaPrinter comma("  ", "");
 816     dumpExpressionStack(comma, expressionStack);
 817     dataLogLn("\n");
 818 }
 819
 820 static std::unique_ptr<B3::Compilation> createJSToWasmWrapper(VM& vm, WasmInternalFunction& function, const Signature* signature, MacroAssemblerCodePtr mainFunction, const MemoryInformation& memory)
 821 {
 822     Procedure proc;
 823     BasicBlock* block = proc.addBlock();
 824
 825     Origin origin;
 826
 827     jscCallingConvention().setupFrameInPrologue(&function.jsToWasmCalleeMoveLocation, proc, origin, block);
 828
 829     if (!ASSERT_DISABLED) {
 830         // This should be guaranteed by our JS wrapper that handles calls to us.
 831         // Just prevent against crazy when ASSERT is enabled.
 832         Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
 833         Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, origin, CallFrameSlot::argumentCount * sizeof(Register));
 834         Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, origin,
 835             block->appendNew<Value>(proc, Add, origin, framePointer, offSetOfArgumentCount));
 836
 837         Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, origin, signature->arguments.size());
 838
 839         CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, origin,
 840             block->appendNew<Value>(proc, Above, origin, expectedArgumentCount, argumentCount));
 841
 842         argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 843             jit.breakpoint();
 844         });
 845     }
 846
 847     // Move memory values to the approriate places, if needed.
 848     Value* baseMemory = nullptr;
 849     Vector<Value*> sizes;
 850     if (!!memory) {
 851         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
 852             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
 853         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 854             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
 855         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
 856         for (auto info : memory.pinnedRegisters().sizeRegisters) {
 857             sizes.append(block->appendNew<Value>(proc, Sub, origin, size,
 858                 block->appendNew<Const32Value>(proc, origin, info.sizeOffset)));
 859         }
 860     }
 861
 862     // Get our arguments.
 863     Vector<Value*> arguments;
 864     jscCallingConvention().loadArguments(signature->arguments, proc, block, origin, [&] (Value* argument, unsigned) {
 865         arguments.append(argument);
 866     });
 867
 868     // Move the arguments into place.
 869     Value* result = wasmCallingConvention().setupCall(proc, block, origin, arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
 870         if (!!memory) {
 871             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
 872             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
 873             for (unsigned i = 0; i < sizes.size(); ++i)
 874                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
 875         }
 876
 877         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 878             AllowMacroScratchRegisterUsage allowScratch(jit);
 879
 880             CCallHelpers::Call call = jit.call();
 881             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 882                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
 883             });
 884         });
 885     });
 886
 887     // Return the result, if needed.
 888     switch (signature->returnType) {
 889     case Wasm::Void:
 890         block->appendNewControlValue(proc, B3::Return, origin);
 891         break;
 892     case Wasm::F32:
 893     case Wasm::F64:
 894         result = block->appendNew<Value>(proc, BitwiseCast, origin, result);
 895         FALLTHROUGH;
 896     case Wasm::I32:
 897     case Wasm::I64:
 898         block->appendNewControlValue(proc, B3::Return, origin, result);
 899         break;
 900     case Wasm::Func:
 901     case Wasm::Anyfunc:
 902         RELEASE_ASSERT_NOT_REACHED();
 903     }
 904
 905     auto jsEntrypoint = std::make_unique<Compilation>(B3::compile(vm, proc));
 906     function.jsToWasmEntrypoint.calleeSaveRegisters = proc.calleeSaveRegisters();
 907     return jsEntrypoint;
 908 }
 909
 910 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace, const ModuleInformation& info, unsigned optLevel)
 911 {
 912     auto result = std::make_unique<WasmInternalFunction>();
 913
 914     Procedure procedure;
 915     B3IRGenerator context(vm, info, procedure, result.get(), unlinkedWasmToWasmCalls, functionIndexSpace);
 916     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace, info);
 917     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
 918
 919     procedure.resetReachability();
 920     validate(procedure, "After parsing:\n");
 921
 922     if (verbose)
 923         dataLog("Pre SSA: ", procedure);
 924     fixSSA(procedure);
 925     if (verbose)
 926         dataLog("Post SSA: ", procedure);
 927
 928     result->wasmEntrypoint.compilation = std::make_unique<B3::Compilation>(B3::compile(vm, procedure, optLevel));
 929     result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
 930
 931     result->jsToWasmEntrypoint.compilation = createJSToWasmWrapper(vm, *result, signature, result->wasmEntrypoint.compilation->code(), info.memory);
 932     return WTFMove(result);
 933 }
 934
 935 // Custom wasm ops. These are the ones too messy to do in wasm.json.
 936
 937 template<>
 938 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
 939 {
 940     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 941     patchpoint->append(arg, ValueRep::SomeRegister);
 942     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 943         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
 944     });
 945     patchpoint->effects = Effects::none();
 946     result = patchpoint;
 947     return { };
 948 }
 949
 950 template<>
 951 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
 952 {
 953     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
 954     patchpoint->append(arg, ValueRep::SomeRegister);
 955     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 956         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
 957     });
 958     patchpoint->effects = Effects::none();
 959     result = patchpoint;
 960     return { };
 961 }
 962
 963 template<>
 964 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
 965 {
 966     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 967     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 968     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
 969     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 970     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
 971     return { };
 972 }
 973
 974 template<>
 975 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
 976 {
 977     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 978     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 979     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
 980     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 981     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
 982     return { };
 983 }
 984
 985 template<>
 986 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
 987 {
 988     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 989     if (isX86())
 990         patchpoint->numGPScratchRegisters = 1;
 991     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
 992     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 993         AllowMacroScratchRegisterUsage allowScratch(jit);
 994 #if CPU(X86_64)
 995         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
 996 #else
 997         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
 998 #endif
 999     });
1000     patchpoint->effects = Effects::none();
1001     result = patchpoint;
1002     return { };
1003 }
1004
1005 template<>
1006 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1007 {
1008     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1009     if (isX86())
1010         patchpoint->numGPScratchRegisters = 1;
1011     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
1012     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1013         AllowMacroScratchRegisterUsage allowScratch(jit);
1014 #if CPU(X86_64)
1015         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1016 #else
1017         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1018 #endif
1019     });
1020     patchpoint->effects = Effects::none();
1021     result = patchpoint;
1022     return { };
1023 }
1024
1025 template<>
1026 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1027 {
1028     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1029     patchpoint->append(arg, ValueRep::SomeRegister);
1030     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1031         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1032     });
1033     patchpoint->effects = Effects::none();
1034     result = patchpoint;
1035     return { };
1036 }
1037
1038 template<>
1039 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1040 {
1041     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1042     patchpoint->append(arg, ValueRep::SomeRegister);
1043     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1044         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1045     });
1046     patchpoint->effects = Effects::none();
1047     result = patchpoint;
1048     return { };
1049 }
1050
1051 template<>
1052 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1053 {
1054     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1055     patchpoint->append(arg, ValueRep::SomeRegister);
1056     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1057         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1058     });
1059     patchpoint->effects = Effects::none();
1060     result = patchpoint;
1061     return { };
1062 }
1063
1064 template<>
1065 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1066 {
1067     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1068     patchpoint->append(arg, ValueRep::SomeRegister);
1069     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1070         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1071     });
1072     patchpoint->effects = Effects::none();
1073     result = patchpoint;
1074     return { };
1075 }
1076
1077 template<>
1078 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1079 {
1080     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
1081     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
1082     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1083         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1084         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1085     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1086     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1087     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1088         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1089     });
1090     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1091     patchpoint->append(arg, ValueRep::SomeRegister);
1092     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1093         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1094     });
1095     patchpoint->effects = Effects::none();
1096     result = patchpoint;
1097     return { };
1098 }
1099
1100 template<>
1101 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1102 {
1103     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
1104     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
1105     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1106         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1107         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1108     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1109     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1110     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1111         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1112     });
1113     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1114     patchpoint->append(arg, ValueRep::SomeRegister);
1115     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1116         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1117     });
1118     patchpoint->effects = Effects::none();
1119     result = patchpoint;
1120     return { };
1121 }
1122
1123
1124 template<>
1125 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1126 {
1127     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1128     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1129     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1130         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1131         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1132     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1133     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1134     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1135         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1136     });
1137     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1138     patchpoint->append(arg, ValueRep::SomeRegister);
1139     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1140         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1141     });
1142     patchpoint->effects = Effects::none();
1143     result = patchpoint;
1144     return { };
1145 }
1146
1147 template<>
1148 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1149 {
1150     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1151     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1152     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1153         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1154         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1155     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1156     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1157     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1158         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1159     });
1160     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1161     patchpoint->append(arg, ValueRep::SomeRegister);
1162     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1163         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1164     });
1165     patchpoint->effects = Effects::none();
1166     result = patchpoint;
1167     return { };
1168 }
1169
1170 template<>
1171 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1172 {
1173     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1174     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1175     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1176         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1177         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1178     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1179     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1180     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1181         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1182     });
1183     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1184     patchpoint->append(arg, ValueRep::SomeRegister);
1185     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1186         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1187     });
1188     patchpoint->effects = Effects::none();
1189     result = patchpoint;
1190     return { };
1191 }
1192
1193 template<>
1194 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1195 {
1196     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1197     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1198     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1199         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1200         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1201     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1202     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1203     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1204         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1205     });
1206
1207     Value* constant;
1208     if (isX86()) {
1209         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1210         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1211         // so we can pool them if needed.
1212         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1213     }
1214     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1215     patchpoint->append(arg, ValueRep::SomeRegister);
1216     if (isX86()) {
1217         patchpoint->append(constant, ValueRep::SomeRegister);
1218         patchpoint->numFPScratchRegisters = 1;
1219     }
1220     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1221         AllowMacroScratchRegisterUsage allowScratch(jit);
1222         FPRReg scratch = InvalidFPRReg;
1223         FPRReg constant = InvalidFPRReg;
1224         if (isX86()) {
1225             scratch = params.fpScratch(0);
1226             constant = params[2].fpr();
1227         }
1228         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1229     });
1230     patchpoint->effects = Effects::none();
1231     result = patchpoint;
1232     return { };
1233 }
1234
1235 template<>
1236 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1237 {
1238     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1239     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1240     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1241         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1242         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1243     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1244     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1245     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1246         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1247     });
1248     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1249     patchpoint->append(arg, ValueRep::SomeRegister);
1250     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1251         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1252     });
1253     patchpoint->effects = Effects::none();
1254     result = patchpoint;
1255     return { };
1256 }
1257
1258 template<>
1259 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1260 {
1261     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1262     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1263     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1264         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1265         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1266     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1267     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1268     trap->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams&) {
1269         this->emitExceptionCheck(jit, ExceptionType::OutOfBoundsTrunc);
1270     });
1271
1272     Value* constant;
1273     if (isX86()) {
1274         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1275         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1276         // so we can pool them if needed.
1277         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1278     }
1279     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1280     patchpoint->append(arg, ValueRep::SomeRegister);
1281     if (isX86()) {
1282         patchpoint->append(constant, ValueRep::SomeRegister);
1283         patchpoint->numFPScratchRegisters = 1;
1284     }
1285     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1286         AllowMacroScratchRegisterUsage allowScratch(jit);
1287         FPRReg scratch = InvalidFPRReg;
1288         FPRReg constant = InvalidFPRReg;
1289         if (isX86()) {
1290             scratch = params.fpScratch(0);
1291             constant = params[2].fpr();
1292         }
1293         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1294     });
1295     patchpoint->effects = Effects::none();
1296     result = patchpoint;
1297     return { };
1298 }
1299
1300 } } // namespace JSC::Wasm
1301
1302 #include "WasmB3IRGeneratorInlines.h"
1303
1304 #endif // ENABLE(WEBASSEMBLY)