Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp

   1 /*
   2  * Copyright (C) 2016 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "WasmB3IRGenerator.h"
  28
  29 #if ENABLE(WEBASSEMBLY)
  30
  31 #include "B3BasicBlockInlines.h"
  32 #include "B3CCallValue.h"
  33 #include "B3ConstPtrValue.h"
  34 #include "B3FixSSA.h"
  35 #include "B3StackmapGenerationParams.h"
  36 #include "B3SwitchValue.h"
  37 #include "B3Validate.h"
  38 #include "B3ValueInlines.h"
  39 #include "B3Variable.h"
  40 #include "B3VariableValue.h"
  41 #include "B3WasmAddressValue.h"
  42 #include "B3WasmBoundsCheckValue.h"
  43 #include "VirtualRegister.h"
  44 #include "WasmCallingConvention.h"
  45 #include "WasmFunctionParser.h"
  46 #include "WasmMemory.h"
  47 #include <wtf/Optional.h>
  48
  49 void dumpProcedure(void* ptr)
  50 {
  51     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
  52     proc->dump(WTF::dataFile());
  53 }
  54
  55 namespace JSC { namespace Wasm {
  56
  57 using namespace B3;
  58
  59 namespace {
  60 const bool verbose = false;
  61 }
  62
  63 class B3IRGenerator {
  64 public:
  65     struct ControlData {
  66         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
  67             : blockType(type)
  68             , continuation(continuation)
  69             , special(special)
  70         {
  71             if (signature != Void)
  72                 result.append(proc.addVariable(toB3Type(signature)));
  73         }
  74
  75         ControlData()
  76         {
  77         }
  78
  79         void dump(PrintStream& out) const
  80         {
  81             switch (type()) {
  82             case BlockType::If:
  83                 out.print("If:    ");
  84                 break;
  85             case BlockType::Block:
  86                 out.print("Block: ");
  87                 break;
  88             case BlockType::Loop:
  89                 out.print("Loop:  ");
  90                 break;
  91             }
  92             out.print("Continuation: ", *continuation, ", Special: ");
  93             if (special)
  94                 out.print(*special);
  95             else
  96                 out.print("None");
  97         }
  98
  99         BlockType type() const { return blockType; }
 100
 101         bool hasNonVoidSignature() const { return result.size(); }
 102
 103         BasicBlock* targetBlockForBranch()
 104         {
 105             if (type() == BlockType::Loop)
 106                 return special;
 107             return continuation;
 108         }
 109
 110         void convertIfToBlock()
 111         {
 112             ASSERT(type() == BlockType::If);
 113             blockType = BlockType::Block;
 114             special = nullptr;
 115         }
 116
 117     private:
 118         friend class B3IRGenerator;
 119         BlockType blockType;
 120         BasicBlock* continuation;
 121         BasicBlock* special;
 122         Vector<Variable*, 1> result;
 123     };
 124
 125     typedef Value* ExpressionType;
 126     typedef ControlData ControlType;
 127     typedef Vector<ExpressionType, 1> ExpressionList;
 128     typedef Vector<Variable*, 1> ResultList;
 129     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
 130
 131     static constexpr ExpressionType emptyExpression = nullptr;
 132
 133     B3IRGenerator(MemoryInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&);
 134
 135     bool WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
 136     bool WARN_UNUSED_RETURN addLocal(Type, uint32_t);
 137     ExpressionType addConstant(Type, uint64_t);
 138
 139     // Locals
 140     bool WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
 141     bool WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
 142
 143     // Memory
 144     bool WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
 145     bool WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
 146
 147     // Basic operators
 148     template<OpType>
 149     bool WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
 150     template<OpType>
 151     bool WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
 152     bool WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
 153
 154     // Control flow
 155     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
 156     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
 157     bool WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
 158     bool WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
 159     bool WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
 160
 161     bool WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
 162     bool WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
 163     bool WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
 164     bool WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
 165     bool WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
 166
 167     bool WARN_UNUSED_RETURN addCall(unsigned calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 168
 169     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
 170
 171     void setErrorMessage(String&&) { UNREACHABLE_FOR_PLATFORM(); }
 172
 173 private:
 174     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
 175     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
 176     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
 177
 178     void unify(Variable* target, const ExpressionType source);
 179     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
 180     Value* zeroForType(Type);
 181
 182     Procedure& m_proc;
 183     BasicBlock* m_currentBlock;
 184     Vector<Variable*> m_locals;
 185     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
 186     GPRReg m_memoryBaseGPR;
 187     GPRReg m_memorySizeGPR;
 188     Value* m_zeroValues[numTypes];
 189 };
 190
 191 B3IRGenerator::B3IRGenerator(MemoryInformation& memory, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls)
 192     : m_proc(procedure)
 193     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
 194 {
 195     m_currentBlock = m_proc.addBlock();
 196
 197     for (unsigned i = 0; i < numTypes; ++i) {
 198         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
 199         case B3::Int32:
 200         case B3::Int64:
 201         case B3::Float:
 202         case B3::Double:
 203             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
 204             break;
 205         case B3::Void:
 206             m_zeroValues[i] = nullptr;
 207             break;
 208         }
 209     }
 210
 211     if (!!memory) {
 212         m_memoryBaseGPR = memory.pinnedRegisters().baseMemoryPointer;
 213         m_proc.pinRegister(m_memoryBaseGPR);
 214         ASSERT(!memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
 215         m_memorySizeGPR = memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
 216         for (const PinnedSizeRegisterInfo& info : memory.pinnedRegisters().sizeRegisters)
 217             m_proc.pinRegister(info.sizeRegister);
 218
 219         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
 220             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
 221             // FIXME: This should unwind the stack and throw a JS exception. See: https://bugs.webkit.org/show_bug.cgi?id=163351
 222             jit.breakpoint();
 223         });
 224     }
 225
 226     wasmCallingConvention().setupFrameInPrologue(compilation, m_proc, Origin(), m_currentBlock);
 227 }
 228
 229 Value* B3IRGenerator::zeroForType(Type type)
 230 {
 231     ASSERT(type != Void);
 232     Value* zeroValue = m_zeroValues[linearizeType(type)];
 233     ASSERT(zeroValue);
 234     return zeroValue;
 235 }
 236
 237 bool B3IRGenerator::addLocal(Type type, uint32_t count)
 238 {
 239     if (!m_locals.tryReserveCapacity(m_locals.size() + count))
 240         return false;
 241
 242     for (uint32_t i = 0; i < count; ++i) {
 243         Variable* local = m_proc.addVariable(toB3Type(type));
 244         m_locals.uncheckedAppend(local);
 245         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
 246     }
 247     return true;
 248 }
 249
 250 bool B3IRGenerator::addArguments(const Vector<Type>& types)
 251 {
 252     ASSERT(!m_locals.size());
 253     if (!m_locals.tryReserveCapacity(types.size()))
 254         return false;
 255
 256     m_locals.grow(types.size());
 257     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
 258         [&] (ExpressionType argument, unsigned i) {
 259             Variable* argumentVariable = m_proc.addVariable(argument->type());
 260             m_locals[i] = argumentVariable;
 261             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
 262         });
 263     return true;
 264 }
 265
 266 bool B3IRGenerator::getLocal(uint32_t index, ExpressionType& result)
 267 {
 268     ASSERT(m_locals[index]);
 269     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
 270     return true;
 271 }
 272
 273 bool B3IRGenerator::setLocal(uint32_t index, ExpressionType value)
 274 {
 275     ASSERT(m_locals[index]);
 276     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
 277     return true;
 278 }
 279
 280 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
 281 {
 282     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
 283     ASSERT(sizeOfOperation + offset > offset);
 284     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
 285     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
 286     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
 287 }
 288
 289 inline uint32_t sizeOfLoadOp(LoadOpType op)
 290 {
 291     switch (op) {
 292     case LoadOpType::I32Load8S:
 293     case LoadOpType::I32Load8U:
 294     case LoadOpType::I64Load8S:
 295     case LoadOpType::I64Load8U:
 296         return 1;
 297     case LoadOpType::I32Load16S:
 298     case LoadOpType::I64Load16S:
 299         return 2;
 300     case LoadOpType::I32Load:
 301     case LoadOpType::I64Load32S:
 302     case LoadOpType::I64Load32U:
 303     case LoadOpType::F32Load:
 304         return 4;
 305     case LoadOpType::I64Load:
 306     case LoadOpType::F64Load:
 307         return 8;
 308     case LoadOpType::I32Load16U:
 309     case LoadOpType::I64Load16U:
 310         break;
 311     }
 312     RELEASE_ASSERT_NOT_REACHED();
 313 }
 314
 315 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
 316 {
 317     switch (op) {
 318     case LoadOpType::I32Load8S: {
 319         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 320     }
 321
 322     case LoadOpType::I64Load8S: {
 323         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 324         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 325     }
 326
 327     case LoadOpType::I32Load8U: {
 328         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 329     }
 330
 331     case LoadOpType::I64Load8U: {
 332         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 333         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 334     }
 335
 336     case LoadOpType::I32Load16S: {
 337         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 338     }
 339     case LoadOpType::I64Load16S: {
 340         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 341         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 342     }
 343
 344     case LoadOpType::I32Load: {
 345         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 346     }
 347
 348     case LoadOpType::I64Load32U: {
 349         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 350         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 351     }
 352
 353     case LoadOpType::I64Load32S: {
 354         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 355         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 356     }
 357
 358     case LoadOpType::I64Load: {
 359         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
 360     }
 361
 362     case LoadOpType::F32Load: {
 363         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
 364     }
 365
 366     case LoadOpType::F64Load: {
 367         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
 368     }
 369
 370     // B3 doesn't support Load16Z yet.
 371     case LoadOpType::I32Load16U:
 372     case LoadOpType::I64Load16U:
 373         break;
 374     }
 375     RELEASE_ASSERT_NOT_REACHED();
 376 }
 377
 378 bool B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset)
 379 {
 380     ASSERT(pointer->type() == Int32);
 381
 382     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
 383     return true;
 384 }
 385
 386 inline uint32_t sizeOfStoreOp(StoreOpType op)
 387 {
 388     switch (op) {
 389     case StoreOpType::I32Store8:
 390     case StoreOpType::I64Store8:
 391         return 1;
 392     case StoreOpType::I32Store16:
 393     case StoreOpType::I64Store16:
 394         return 2;
 395     case StoreOpType::I32Store:
 396     case StoreOpType::I64Store32:
 397     case StoreOpType::F32Store:
 398         return 4;
 399     case StoreOpType::I64Store:
 400     case StoreOpType::F64Store:
 401         return 8;
 402     }
 403     RELEASE_ASSERT_NOT_REACHED();
 404 }
 405
 406
 407 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
 408 {
 409     switch (op) {
 410     case StoreOpType::I64Store8:
 411         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 412         FALLTHROUGH;
 413
 414     case StoreOpType::I32Store8:
 415         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
 416         return;
 417
 418     case StoreOpType::I64Store16:
 419         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 420         FALLTHROUGH;
 421
 422     case StoreOpType::I32Store16:
 423         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
 424         return;
 425
 426     case StoreOpType::I64Store32:
 427         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 428         FALLTHROUGH;
 429
 430     case StoreOpType::I64Store:
 431     case StoreOpType::I32Store:
 432     case StoreOpType::F32Store:
 433     case StoreOpType::F64Store:
 434         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
 435         return;
 436     }
 437     RELEASE_ASSERT_NOT_REACHED();
 438 }
 439
 440 bool B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset)
 441 {
 442     ASSERT(pointer->type() == Int32);
 443
 444     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
 445     return true;
 446 }
 447
 448 bool B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result)
 449 {
 450     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
 451     return true;
 452 }
 453
 454 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
 455 {
 456     switch (type) {
 457     case Wasm::I32:
 458         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
 459     case Wasm::I64:
 460         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
 461     case Wasm::F32:
 462         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
 463     case Wasm::F64:
 464         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
 465     case Wasm::Void:
 466     case Wasm::Func:
 467     case Wasm::Anyfunc:
 468         break;
 469     }
 470     RELEASE_ASSERT_NOT_REACHED();
 471     return nullptr;
 472 }
 473
 474 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
 475 {
 476     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
 477 }
 478
 479 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
 480 {
 481     BasicBlock* body = m_proc.addBlock();
 482     BasicBlock* continuation = m_proc.addBlock();
 483     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
 484     body->addPredecessor(m_currentBlock);
 485     m_currentBlock = body;
 486     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
 487 }
 488
 489 bool B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result)
 490 {
 491     // FIXME: This needs to do some kind of stack passing.
 492
 493     BasicBlock* taken = m_proc.addBlock();
 494     BasicBlock* notTaken = m_proc.addBlock();
 495     BasicBlock* continuation = m_proc.addBlock();
 496
 497     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 498     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
 499     taken->addPredecessor(m_currentBlock);
 500     notTaken->addPredecessor(m_currentBlock);
 501
 502     m_currentBlock = taken;
 503     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
 504     return true;
 505 }
 506
 507 bool B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack)
 508 {
 509     unifyValuesWithBlock(currentStack, data.result);
 510     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 511     return addElseToUnreachable(data);
 512 }
 513
 514 bool B3IRGenerator::addElseToUnreachable(ControlData& data)
 515 {
 516     ASSERT(data.type() == BlockType::If);
 517     m_currentBlock = data.special;
 518     data.convertIfToBlock();
 519     return true;
 520 }
 521
 522 bool B3IRGenerator::addReturn(const ExpressionList& returnValues)
 523 {
 524     ASSERT(returnValues.size() <= 1);
 525     if (returnValues.size())
 526         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
 527     else
 528         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
 529     return true;
 530 }
 531
 532 bool B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues)
 533 {
 534     if (data.type() != BlockType::Loop)
 535         unifyValuesWithBlock(returnValues, data.result);
 536
 537     BasicBlock* target = data.targetBlockForBranch();
 538     if (condition) {
 539         BasicBlock* continuation = m_proc.addBlock();
 540         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 541         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
 542         target->addPredecessor(m_currentBlock);
 543         continuation->addPredecessor(m_currentBlock);
 544         m_currentBlock = continuation;
 545     } else {
 546         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
 547         target->addPredecessor(m_currentBlock);
 548     }
 549
 550     return true;
 551 }
 552
 553 bool B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack)
 554 {
 555     for (size_t i = 0; i < targets.size(); ++i)
 556         unifyValuesWithBlock(expressionStack, targets[i]->result);
 557     unifyValuesWithBlock(expressionStack, defaultTarget.result);
 558
 559     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
 560     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
 561     for (size_t i = 0; i < targets.size(); ++i)
 562         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
 563
 564     return true;
 565 }
 566
 567 bool B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack)
 568 {
 569     ControlData& data = entry.controlData;
 570
 571     unifyValuesWithBlock(expressionStack, data.result);
 572     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 573     data.continuation->addPredecessor(m_currentBlock);
 574
 575     return addEndToUnreachable(entry);
 576 }
 577
 578
 579 bool B3IRGenerator::addEndToUnreachable(ControlEntry& entry)
 580 {
 581     ControlData& data = entry.controlData;
 582     m_currentBlock = data.continuation;
 583
 584     if (data.type() == BlockType::If) {
 585         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
 586         m_currentBlock->addPredecessor(data.special);
 587     }
 588
 589     for (Variable* result : data.result)
 590         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
 591
 592     return true;
 593 }
 594
 595 bool B3IRGenerator::addCall(unsigned functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result)
 596 {
 597     ASSERT(signature->arguments.size() == args.size());
 598
 599     Type returnType = signature->returnType;
 600
 601     size_t callIndex = m_unlinkedWasmToWasmCalls.size();
 602     m_unlinkedWasmToWasmCalls.grow(callIndex + 1);
 603     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
 604         [&] (PatchpointValue* patchpoint) {
 605             patchpoint->effects.writesPinned = true;
 606             patchpoint->effects.readsPinned = true;
 607
 608             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 609                 AllowMacroScratchRegisterUsage allowScratch(jit);
 610
 611                 CCallHelpers::Call call = jit.call();
 612
 613                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 614                     m_unlinkedWasmToWasmCalls[callIndex] = { linkBuffer.locationOf(call), functionIndex };
 615                 });
 616             });
 617         });
 618     return true;
 619 }
 620
 621 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
 622 {
 623     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
 624 }
 625
 626 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
 627 {
 628     ASSERT(result.size() <= resultStack.size());
 629
 630     for (size_t i = 0; i < result.size(); ++i)
 631         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
 632 }
 633
 634 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
 635 {
 636     dataLogLn(comma, "ExpressionStack:");
 637     for (const auto& expression : expressionStack)
 638         dataLogLn(comma, *expression);
 639 }
 640
 641 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
 642 {
 643     dataLogLn("Processing Graph:");
 644     dataLog(m_proc);
 645     dataLogLn("With current block:", *m_currentBlock);
 646     dataLogLn("Control stack:");
 647     for (auto& data : controlStack) {
 648         dataLogLn("  ", data.controlData);
 649         if (data.enclosedExpressionStack.size()) {
 650             CommaPrinter comma("    ", "  with ");
 651             dumpExpressionStack(comma, data.enclosedExpressionStack);
 652         }
 653     }
 654
 655     CommaPrinter comma("  ", "");
 656     dumpExpressionStack(comma, expressionStack);
 657     dataLogLn("\n");
 658 }
 659
 660 static std::unique_ptr<Compilation> createJSToWasmWrapper(VM& vm, const Signature* signature, MacroAssemblerCodePtr mainFunction, MemoryInformation& memory)
 661 {
 662     Procedure proc;
 663     BasicBlock* block = proc.addBlock();
 664
 665     // Check argument count is sane.
 666     Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, Origin());
 667     Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, Origin(), CallFrameSlot::argumentCount * sizeof(Register));
 668     Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 669         block->appendNew<Value>(proc, Add, Origin(), framePointer, offSetOfArgumentCount));
 670
 671     Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, Origin(), signature->arguments.size());
 672
 673     CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, Origin(),
 674         block->appendNew<Value>(proc, Above, Origin(), expectedArgumentCount, argumentCount));
 675     argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 676         jit.breakpoint();
 677     });
 678
 679     // Move memory values to the approriate places, if needed.
 680     Value* baseMemory = nullptr;
 681     Vector<Value*> sizes;
 682     if (!!memory) {
 683         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
 684             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
 685         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 686             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
 687         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
 688         for (auto info : memory.pinnedRegisters().sizeRegisters) {
 689             sizes.append(block->appendNew<Value>(proc, Sub, Origin(), size,
 690                 block->appendNew<Const32Value>(proc, Origin(), info.sizeOffset)));
 691         }
 692     }
 693
 694     // Get our arguments.
 695     Vector<Value*> arguments;
 696     jscCallingConvention().loadArguments(signature->arguments, proc, block, Origin(), [&] (Value* argument, unsigned) {
 697         arguments.append(argument);
 698     });
 699
 700     // Move the arguments into place.
 701     Value* result = wasmCallingConvention().setupCall(proc, block, Origin(), arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
 702         if (!!memory) {
 703             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
 704             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
 705             for (unsigned i = 0; i < sizes.size(); ++i)
 706                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
 707         }
 708
 709         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 710             AllowMacroScratchRegisterUsage allowScratch(jit);
 711
 712             CCallHelpers::Call call = jit.call();
 713             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 714                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
 715             });
 716         });
 717     });
 718
 719     // Return the result, if needed.
 720     switch (signature->returnType) {
 721     case Wasm::Void:
 722         block->appendNewControlValue(proc, B3::Return, Origin());
 723         break;
 724     case Wasm::F32:
 725     case Wasm::F64:
 726         result = block->appendNew<Value>(proc, BitwiseCast, Origin(), result);
 727         FALLTHROUGH;
 728     case Wasm::I32:
 729     case Wasm::I64:
 730         block->appendNewControlValue(proc, B3::Return, Origin(), result);
 731         break;
 732     case Wasm::Func:
 733     case Wasm::Anyfunc:
 734         RELEASE_ASSERT_NOT_REACHED();
 735     }
 736
 737     return std::make_unique<Compilation>(vm, proc);
 738 }
 739
 740 std::unique_ptr<WasmInternalFunction> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, MemoryInformation& memory, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const FunctionIndexSpace& functionIndexSpace, unsigned optLevel)
 741 {
 742     auto result = std::make_unique<WasmInternalFunction>();
 743
 744     Procedure procedure;
 745     B3IRGenerator context(memory, procedure, result.get(), unlinkedWasmToWasmCalls);
 746     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace);
 747     if (!parser.parse())
 748         RELEASE_ASSERT_NOT_REACHED();
 749
 750     procedure.resetReachability();
 751     validate(procedure, "After parsing:\n");
 752
 753     if (verbose)
 754         dataLog("Pre SSA: ", procedure);
 755     fixSSA(procedure);
 756     if (verbose)
 757         dataLog("Post SSA: ", procedure);
 758
 759     result->code = std::make_unique<Compilation>(vm, procedure, optLevel);
 760     result->jsToWasmEntryPoint = createJSToWasmWrapper(vm, signature, result->code->code(), memory);
 761     return result;
 762 }
 763
 764 // Custom wasm ops. These are the ones too messy to do in wasm.json.
 765
 766 template<>
 767 bool B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result)
 768 {
 769     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 770     patchpoint->append(arg, ValueRep::SomeRegister);
 771     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 772         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
 773     });
 774     patchpoint->effects = Effects::none();
 775     result = patchpoint;
 776     return true;
 777 }
 778
 779 template<>
 780 bool B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result)
 781 {
 782     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
 783     patchpoint->append(arg, ValueRep::SomeRegister);
 784     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 785         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
 786     });
 787     patchpoint->effects = Effects::none();
 788     result = patchpoint;
 789     return true;
 790 }
 791
 792 template<>
 793 bool B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result)
 794 {
 795     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 796     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 797     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
 798     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 799     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
 800     return true;
 801 }
 802
 803 template<>
 804 bool B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result)
 805 {
 806     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 807     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 808     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
 809     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 810     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
 811     return true;
 812 }
 813
 814 template<>
 815 bool B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result)
 816 {
 817     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 818     if (isX86())
 819         patchpoint->numGPScratchRegisters = 1;
 820     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
 821     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 822         AllowMacroScratchRegisterUsage allowScratch(jit);
 823 #if CPU(X86_64)
 824         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
 825 #else
 826         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
 827 #endif
 828     });
 829     patchpoint->effects = Effects::none();
 830     result = patchpoint;
 831     return true;
 832 }
 833
 834 template<>
 835 bool B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result)
 836 {
 837     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 838     if (isX86())
 839         patchpoint->numGPScratchRegisters = 1;
 840     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
 841     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 842         AllowMacroScratchRegisterUsage allowScratch(jit);
 843 #if CPU(X86_64)
 844         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
 845 #else
 846         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
 847 #endif
 848     });
 849     patchpoint->effects = Effects::none();
 850     result = patchpoint;
 851     return true;
 852 }
 853
 854 template<>
 855 bool B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result)
 856 {
 857     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 858     patchpoint->append(arg, ValueRep::SomeRegister);
 859     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 860         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
 861     });
 862     patchpoint->effects = Effects::none();
 863     result = patchpoint;
 864     return true;
 865 }
 866
 867 template<>
 868 bool B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result)
 869 {
 870     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 871     patchpoint->append(arg, ValueRep::SomeRegister);
 872     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 873         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
 874     });
 875     patchpoint->effects = Effects::none();
 876     result = patchpoint;
 877     return true;
 878 }
 879
 880 template<>
 881 bool B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result)
 882 {
 883     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 884     patchpoint->append(arg, ValueRep::SomeRegister);
 885     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 886         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
 887     });
 888     patchpoint->effects = Effects::none();
 889     result = patchpoint;
 890     return true;
 891 }
 892
 893 template<>
 894 bool B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result)
 895 {
 896     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 897     patchpoint->append(arg, ValueRep::SomeRegister);
 898     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 899         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
 900     });
 901     patchpoint->effects = Effects::none();
 902     result = patchpoint;
 903     return true;
 904 }
 905
 906 template<>
 907 bool B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result)
 908 {
 909     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
 910     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
 911     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 912         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 913         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
 914     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 915     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 916     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 917         jit.breakpoint();
 918     });
 919     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 920     patchpoint->append(arg, ValueRep::SomeRegister);
 921     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 922         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
 923     });
 924     patchpoint->effects = Effects::none();
 925     result = patchpoint;
 926     return true;
 927 }
 928
 929 template<>
 930 bool B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result)
 931 {
 932     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
 933     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
 934     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 935         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 936         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
 937     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 938     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 939     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 940         jit.breakpoint();
 941     });
 942     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 943     patchpoint->append(arg, ValueRep::SomeRegister);
 944     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 945         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
 946     });
 947     patchpoint->effects = Effects::none();
 948     result = patchpoint;
 949     return true;
 950 }
 951
 952
 953 template<>
 954 bool B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result)
 955 {
 956     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
 957     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
 958     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 959         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 960         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
 961     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 962     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 963     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 964         jit.breakpoint();
 965     });
 966     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 967     patchpoint->append(arg, ValueRep::SomeRegister);
 968     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 969         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
 970     });
 971     patchpoint->effects = Effects::none();
 972     result = patchpoint;
 973     return true;
 974 }
 975
 976 template<>
 977 bool B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result)
 978 {
 979     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
 980     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
 981     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 982         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 983         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
 984     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 985     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 986     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 987         jit.breakpoint();
 988     });
 989     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 990     patchpoint->append(arg, ValueRep::SomeRegister);
 991     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 992         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
 993     });
 994     patchpoint->effects = Effects::none();
 995     result = patchpoint;
 996     return true;
 997 }
 998
 999 template<>
1000 bool B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result)
1001 {
1002     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1003     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1004     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1005         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1006         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1007     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1008     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1009     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1010         jit.breakpoint();
1011     });
1012     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1013     patchpoint->append(arg, ValueRep::SomeRegister);
1014     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1015         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1016     });
1017     patchpoint->effects = Effects::none();
1018     result = patchpoint;
1019     return true;
1020 }
1021
1022 template<>
1023 bool B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result)
1024 {
1025     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1026     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1027     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1028         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1029         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1030     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1031     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1032     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1033         jit.breakpoint();
1034     });
1035
1036     Value* constant;
1037     if (isX86()) {
1038         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1039         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1040         // so we can pool them if needed.
1041         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1042     }
1043     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1044     patchpoint->append(arg, ValueRep::SomeRegister);
1045     if (isX86()) {
1046         patchpoint->append(constant, ValueRep::SomeRegister);
1047         patchpoint->numFPScratchRegisters = 1;
1048     }
1049     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1050         AllowMacroScratchRegisterUsage allowScratch(jit);
1051         FPRReg scratch = InvalidFPRReg;
1052         FPRReg constant = InvalidFPRReg;
1053         if (isX86()) {
1054             scratch = params.fpScratch(0);
1055             constant = params[2].fpr();
1056         }
1057         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1058     });
1059     patchpoint->effects = Effects::none();
1060     result = patchpoint;
1061     return true;
1062 }
1063
1064 template<>
1065 bool B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result)
1066 {
1067     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1068     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1069     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1070         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1071         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1072     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1073     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1074     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1075         jit.breakpoint();
1076     });
1077     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1078     patchpoint->append(arg, ValueRep::SomeRegister);
1079     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1080         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1081     });
1082     patchpoint->effects = Effects::none();
1083     result = patchpoint;
1084     return true;
1085 }
1086
1087 template<>
1088 bool B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result)
1089 {
1090     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1091     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1092     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1093         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1094         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1095     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1096     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1097     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1098         jit.breakpoint();
1099     });
1100
1101     Value* constant;
1102     if (isX86()) {
1103         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1104         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1105         // so we can pool them if needed.
1106         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1107     }
1108     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1109     patchpoint->append(arg, ValueRep::SomeRegister);
1110     if (isX86()) {
1111         patchpoint->append(constant, ValueRep::SomeRegister);
1112         patchpoint->numFPScratchRegisters = 1;
1113     }
1114     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1115         AllowMacroScratchRegisterUsage allowScratch(jit);
1116         FPRReg scratch = InvalidFPRReg;
1117         FPRReg constant = InvalidFPRReg;
1118         if (isX86()) {
1119             scratch = params.fpScratch(0);
1120             constant = params[2].fpr();
1121         }
1122         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1123     });
1124     patchpoint->effects = Effects::none();
1125     result = patchpoint;
1126     return true;
1127 }
1128
1129 } } // namespace JSC::Wasm
1130
1131 #include "WasmB3IRGeneratorInlines.h"
1132
1133 #endif // ENABLE(WEBASSEMBLY)