Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp

   1 /*
   2  * Copyright (C) 2016 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "WasmB3IRGenerator.h"
  28
  29 #if ENABLE(WEBASSEMBLY)
  30
  31 #include "B3BasicBlockInlines.h"
  32 #include "B3CCallValue.h"
  33 #include "B3ConstPtrValue.h"
  34 #include "B3FixSSA.h"
  35 #include "B3StackmapGenerationParams.h"
  36 #include "B3SwitchValue.h"
  37 #include "B3Validate.h"
  38 #include "B3ValueInlines.h"
  39 #include "B3Variable.h"
  40 #include "B3VariableValue.h"
  41 #include "B3WasmAddressValue.h"
  42 #include "B3WasmBoundsCheckValue.h"
  43 #include "VirtualRegister.h"
  44 #include "WasmCallingConvention.h"
  45 #include "WasmFunctionParser.h"
  46 #include "WasmMemory.h"
  47 #include <wtf/Optional.h>
  48
  49 void dumpProcedure(void* ptr)
  50 {
  51     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
  52     proc->dump(WTF::dataFile());
  53 }
  54
  55 namespace JSC { namespace Wasm {
  56
  57 using namespace B3;
  58
  59 namespace {
  60 const bool verbose = false;
  61 }
  62
  63 class B3IRGenerator {
  64 public:
  65     struct ControlData {
  66         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
  67             : blockType(type)
  68             , continuation(continuation)
  69             , special(special)
  70         {
  71             if (signature != Void)
  72                 result.append(proc.addVariable(toB3Type(signature)));
  73         }
  74
  75         ControlData()
  76         {
  77         }
  78
  79         void dump(PrintStream& out) const
  80         {
  81             switch (type()) {
  82             case BlockType::If:
  83                 out.print("If:    ");
  84                 break;
  85             case BlockType::Block:
  86                 out.print("Block: ");
  87                 break;
  88             case BlockType::Loop:
  89                 out.print("Loop:  ");
  90                 break;
  91             }
  92             out.print("Continuation: ", *continuation, ", Special: ");
  93             if (special)
  94                 out.print(*special);
  95             else
  96                 out.print("None");
  97         }
  98
  99         BlockType type() const { return blockType; }
 100
 101         bool hasNonVoidSignature() const { return result.size(); }
 102
 103         BasicBlock* targetBlockForBranch()
 104         {
 105             if (type() == BlockType::Loop)
 106                 return special;
 107             return continuation;
 108         }
 109
 110         void convertIfToBlock()
 111         {
 112             ASSERT(type() == BlockType::If);
 113             blockType = BlockType::Block;
 114             special = nullptr;
 115         }
 116
 117     private:
 118         friend class B3IRGenerator;
 119         BlockType blockType;
 120         BasicBlock* continuation;
 121         BasicBlock* special;
 122         Vector<Variable*, 1> result;
 123     };
 124
 125     typedef Value* ExpressionType;
 126     typedef ControlData ControlType;
 127     typedef Vector<ExpressionType, 1> ExpressionList;
 128     typedef Vector<Variable*, 1> ResultList;
 129     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
 130
 131     static constexpr ExpressionType emptyExpression = nullptr;
 132
 133     B3IRGenerator(Memory*, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&);
 134
 135     bool WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
 136     bool WARN_UNUSED_RETURN addLocal(Type, uint32_t);
 137     ExpressionType addConstant(Type, uint64_t);
 138
 139     // Locals
 140     bool WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
 141     bool WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
 142
 143     // Memory
 144     bool WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
 145     bool WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
 146
 147     // Basic operators
 148     template<OpType>
 149     bool WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
 150     template<OpType>
 151     bool WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
 152     bool WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
 153
 154     // Control flow
 155     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
 156     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
 157     bool WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
 158     bool WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
 159     bool WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
 160
 161     bool WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
 162     bool WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
 163     bool WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
 164     bool WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
 165     bool WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
 166
 167     bool WARN_UNUSED_RETURN addCall(unsigned calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 168
 169     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
 170
 171     void setErrorMessage(String&&) { UNREACHABLE_FOR_PLATFORM(); }
 172
 173 private:
 174     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
 175     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
 176     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
 177
 178     void unify(Variable* target, const ExpressionType source);
 179     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
 180     Value* zeroForType(Type);
 181
 182     Memory* m_memory;
 183     Procedure& m_proc;
 184     BasicBlock* m_currentBlock;
 185     Vector<Variable*> m_locals;
 186     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
 187     GPRReg m_memoryBaseGPR;
 188     GPRReg m_memorySizeGPR;
 189     Value* m_zeroValues[numTypes];
 190 };
 191
 192 B3IRGenerator::B3IRGenerator(Memory* memory, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls)
 193     : m_memory(memory)
 194     , m_proc(procedure)
 195     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
 196 {
 197     m_currentBlock = m_proc.addBlock();
 198
 199     for (unsigned i = 0; i < numTypes; ++i) {
 200         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
 201         case B3::Int32:
 202         case B3::Int64:
 203         case B3::Float:
 204         case B3::Double:
 205             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
 206             break;
 207         case B3::Void:
 208             m_zeroValues[i] = nullptr;
 209             break;
 210         }
 211     }
 212
 213     if (m_memory) {
 214         m_memoryBaseGPR = m_memory->pinnedRegisters().baseMemoryPointer;
 215         m_proc.pinRegister(m_memoryBaseGPR);
 216         ASSERT(!m_memory->pinnedRegisters().sizeRegisters[0].sizeOffset);
 217         m_memorySizeGPR = m_memory->pinnedRegisters().sizeRegisters[0].sizeRegister;
 218         for (const PinnedSizeRegisterInfo& info : m_memory->pinnedRegisters().sizeRegisters)
 219             m_proc.pinRegister(info.sizeRegister);
 220
 221         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
 222             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
 223             // FIXME: This should unwind the stack and throw a JS exception. See: https://bugs.webkit.org/show_bug.cgi?id=163351
 224             jit.breakpoint();
 225         });
 226     }
 227
 228     wasmCallingConvention().setupFrameInPrologue(compilation, m_proc, Origin(), m_currentBlock);
 229 }
 230
 231 Value* B3IRGenerator::zeroForType(Type type)
 232 {
 233     ASSERT(type != Void);
 234     Value* zeroValue = m_zeroValues[linearizeType(type)];
 235     ASSERT(zeroValue);
 236     return zeroValue;
 237 }
 238
 239 bool B3IRGenerator::addLocal(Type type, uint32_t count)
 240 {
 241     if (!m_locals.tryReserveCapacity(m_locals.size() + count))
 242         return false;
 243
 244     for (uint32_t i = 0; i < count; ++i) {
 245         Variable* local = m_proc.addVariable(toB3Type(type));
 246         m_locals.uncheckedAppend(local);
 247         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
 248     }
 249     return true;
 250 }
 251
 252 bool B3IRGenerator::addArguments(const Vector<Type>& types)
 253 {
 254     ASSERT(!m_locals.size());
 255     if (!m_locals.tryReserveCapacity(types.size()))
 256         return false;
 257
 258     m_locals.grow(types.size());
 259     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
 260         [&] (ExpressionType argument, unsigned i) {
 261             Variable* argumentVariable = m_proc.addVariable(argument->type());
 262             m_locals[i] = argumentVariable;
 263             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
 264         });
 265     return true;
 266 }
 267
 268 bool B3IRGenerator::getLocal(uint32_t index, ExpressionType& result)
 269 {
 270     ASSERT(m_locals[index]);
 271     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
 272     return true;
 273 }
 274
 275 bool B3IRGenerator::setLocal(uint32_t index, ExpressionType value)
 276 {
 277     ASSERT(m_locals[index]);
 278     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
 279     return true;
 280 }
 281
 282 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
 283 {
 284     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
 285     ASSERT(sizeOfOperation + offset > offset);
 286     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
 287     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
 288     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
 289 }
 290
 291 inline uint32_t sizeOfLoadOp(LoadOpType op)
 292 {
 293     switch (op) {
 294     case LoadOpType::I32Load8S:
 295     case LoadOpType::I32Load8U:
 296     case LoadOpType::I64Load8S:
 297     case LoadOpType::I64Load8U:
 298         return 1;
 299     case LoadOpType::I32Load16S:
 300     case LoadOpType::I64Load16S:
 301         return 2;
 302     case LoadOpType::I32Load:
 303     case LoadOpType::I64Load32S:
 304     case LoadOpType::I64Load32U:
 305         return 4;
 306     case LoadOpType::I64Load:
 307         return 8;
 308     case LoadOpType::I32Load16U:
 309     case LoadOpType::I64Load16U:
 310     case LoadOpType::F32Load:
 311     case LoadOpType::F64Load:
 312         break;
 313     }
 314     RELEASE_ASSERT_NOT_REACHED();
 315 }
 316
 317 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
 318 {
 319     switch (op) {
 320     case LoadOpType::I32Load8S: {
 321         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 322     }
 323
 324     case LoadOpType::I64Load8S: {
 325         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 326         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 327     }
 328
 329     case LoadOpType::I32Load8U: {
 330         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 331     }
 332
 333     case LoadOpType::I64Load8U: {
 334         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 335         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 336     }
 337
 338     case LoadOpType::I32Load16S: {
 339         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 340     }
 341     case LoadOpType::I64Load16S: {
 342         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 343         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 344     }
 345
 346     case LoadOpType::I32Load: {
 347         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 348     }
 349
 350     case LoadOpType::I64Load32U: {
 351         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 352         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 353     }
 354
 355     case LoadOpType::I64Load32S: {
 356         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 357         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 358     }
 359
 360     case LoadOpType::I64Load: {
 361         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
 362     }
 363
 364     case LoadOpType::F32Load: {
 365         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
 366     }
 367
 368     case LoadOpType::F64Load: {
 369         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
 370     }
 371
 372     // B3 doesn't support Load16Z yet.
 373     case LoadOpType::I32Load16U:
 374     case LoadOpType::I64Load16U:
 375         break;
 376     }
 377     RELEASE_ASSERT_NOT_REACHED();
 378 }
 379
 380 bool B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset)
 381 {
 382     ASSERT(pointer->type() == Int32);
 383
 384     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
 385     return true;
 386 }
 387
 388 inline uint32_t sizeOfStoreOp(StoreOpType op)
 389 {
 390     switch (op) {
 391     case StoreOpType::I32Store8:
 392     case StoreOpType::I64Store8:
 393         return 1;
 394     case StoreOpType::I32Store16:
 395     case StoreOpType::I64Store16:
 396         return 2;
 397     case StoreOpType::I32Store:
 398     case StoreOpType::I64Store32:
 399     case StoreOpType::F32Store:
 400         return 4;
 401     case StoreOpType::I64Store:
 402     case StoreOpType::F64Store:
 403         return 8;
 404     }
 405     RELEASE_ASSERT_NOT_REACHED();
 406 }
 407
 408
 409 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
 410 {
 411     switch (op) {
 412     case StoreOpType::I64Store8:
 413         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 414         FALLTHROUGH;
 415
 416     case StoreOpType::I32Store8:
 417         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
 418         return;
 419
 420     case StoreOpType::I64Store16:
 421         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 422         FALLTHROUGH;
 423
 424     case StoreOpType::I32Store16:
 425         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
 426         return;
 427
 428     case StoreOpType::I64Store32:
 429         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 430         FALLTHROUGH;
 431
 432     case StoreOpType::I64Store:
 433     case StoreOpType::I32Store:
 434     case StoreOpType::F32Store:
 435     case StoreOpType::F64Store:
 436         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
 437         return;
 438     }
 439     RELEASE_ASSERT_NOT_REACHED();
 440 }
 441
 442 bool B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset)
 443 {
 444     ASSERT(pointer->type() == Int32);
 445
 446     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
 447     return true;
 448 }
 449
 450 bool B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result)
 451 {
 452     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
 453     return true;
 454 }
 455
 456 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
 457 {
 458     switch (type) {
 459     case Wasm::I32:
 460         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
 461     case Wasm::I64:
 462         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
 463     case Wasm::F32:
 464         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
 465     case Wasm::F64:
 466         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
 467     case Wasm::Void:
 468     case Wasm::Func:
 469     case Wasm::Anyfunc:
 470         break;
 471     }
 472     RELEASE_ASSERT_NOT_REACHED();
 473     return nullptr;
 474 }
 475
 476 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
 477 {
 478     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
 479 }
 480
 481 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
 482 {
 483     BasicBlock* body = m_proc.addBlock();
 484     BasicBlock* continuation = m_proc.addBlock();
 485     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
 486     body->addPredecessor(m_currentBlock);
 487     m_currentBlock = body;
 488     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
 489 }
 490
 491 bool B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result)
 492 {
 493     // FIXME: This needs to do some kind of stack passing.
 494
 495     BasicBlock* taken = m_proc.addBlock();
 496     BasicBlock* notTaken = m_proc.addBlock();
 497     BasicBlock* continuation = m_proc.addBlock();
 498
 499     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 500     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
 501     taken->addPredecessor(m_currentBlock);
 502     notTaken->addPredecessor(m_currentBlock);
 503
 504     m_currentBlock = taken;
 505     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
 506     return true;
 507 }
 508
 509 bool B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack)
 510 {
 511     unifyValuesWithBlock(currentStack, data.result);
 512     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 513     return addElseToUnreachable(data);
 514 }
 515
 516 bool B3IRGenerator::addElseToUnreachable(ControlData& data)
 517 {
 518     ASSERT(data.type() == BlockType::If);
 519     m_currentBlock = data.special;
 520     data.convertIfToBlock();
 521     return true;
 522 }
 523
 524 bool B3IRGenerator::addReturn(const ExpressionList& returnValues)
 525 {
 526     ASSERT(returnValues.size() <= 1);
 527     if (returnValues.size())
 528         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
 529     else
 530         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
 531     return true;
 532 }
 533
 534 bool B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues)
 535 {
 536     if (data.type() != BlockType::Loop)
 537         unifyValuesWithBlock(returnValues, data.result);
 538
 539     BasicBlock* target = data.targetBlockForBranch();
 540     if (condition) {
 541         BasicBlock* continuation = m_proc.addBlock();
 542         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 543         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
 544         target->addPredecessor(m_currentBlock);
 545         continuation->addPredecessor(m_currentBlock);
 546         m_currentBlock = continuation;
 547     } else {
 548         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
 549         target->addPredecessor(m_currentBlock);
 550     }
 551
 552     return true;
 553 }
 554
 555 bool B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack)
 556 {
 557     for (size_t i = 0; i < targets.size(); ++i)
 558         unifyValuesWithBlock(expressionStack, targets[i]->result);
 559     unifyValuesWithBlock(expressionStack, defaultTarget.result);
 560
 561     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
 562     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
 563     for (size_t i = 0; i < targets.size(); ++i)
 564         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
 565
 566     return true;
 567 }
 568
 569 bool B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack)
 570 {
 571     ControlData& data = entry.controlData;
 572
 573     unifyValuesWithBlock(expressionStack, data.result);
 574     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 575     data.continuation->addPredecessor(m_currentBlock);
 576
 577     return addEndToUnreachable(entry);
 578 }
 579
 580
 581 bool B3IRGenerator::addEndToUnreachable(ControlEntry& entry)
 582 {
 583     ControlData& data = entry.controlData;
 584     m_currentBlock = data.continuation;
 585
 586     if (data.type() == BlockType::If) {
 587         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
 588         m_currentBlock->addPredecessor(data.special);
 589     }
 590
 591     for (Variable* result : data.result)
 592         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
 593
 594     return true;
 595 }
 596
 597 bool B3IRGenerator::addCall(unsigned functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result)
 598 {
 599     ASSERT(signature->arguments.size() == args.size());
 600
 601     Type returnType = signature->returnType;
 602
 603     size_t callIndex = m_unlinkedWasmToWasmCalls.size();
 604     m_unlinkedWasmToWasmCalls.grow(callIndex + 1);
 605     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
 606         [&] (PatchpointValue* patchpoint) {
 607             patchpoint->effects.writesPinned = true;
 608             patchpoint->effects.readsPinned = true;
 609
 610             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 611                 AllowMacroScratchRegisterUsage allowScratch(jit);
 612
 613                 CCallHelpers::Call call = jit.call();
 614
 615                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 616                     m_unlinkedWasmToWasmCalls[callIndex] = { linkBuffer.locationOf(call), functionIndex };
 617                 });
 618             });
 619         });
 620     return true;
 621 }
 622
 623 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
 624 {
 625     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
 626 }
 627
 628 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
 629 {
 630     ASSERT(result.size() <= resultStack.size());
 631
 632     for (size_t i = 0; i < result.size(); ++i)
 633         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
 634 }
 635
 636 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
 637 {
 638     dataLogLn(comma, "ExpressionStack:");
 639     for (const auto& expression : expressionStack)
 640         dataLogLn(comma, *expression);
 641 }
 642
 643 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
 644 {
 645     dataLogLn("Processing Graph:");
 646     dataLog(m_proc);
 647     dataLogLn("With current block:", *m_currentBlock);
 648     dataLogLn("Control stack:");
 649     for (auto& data : controlStack) {
 650         dataLogLn("  ", data.controlData);
 651         if (data.enclosedExpressionStack.size()) {
 652             CommaPrinter comma("    ", "  with ");
 653             dumpExpressionStack(comma, data.enclosedExpressionStack);
 654         }
 655     }
 656
 657     CommaPrinter comma("  ", "");
 658     dumpExpressionStack(comma, expressionStack);
 659     dataLogLn("\n");
 660 }
 661
 662 static std::unique_ptr<Compilation> createJSToWasmWrapper(VM& vm, const Signature* signature, MacroAssemblerCodePtr mainFunction, Memory* memory)
 663 {
 664     Procedure proc;
 665     BasicBlock* block = proc.addBlock();
 666
 667     // Check argument count is sane.
 668     Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, Origin());
 669     Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, Origin(), CallFrameSlot::argumentCount * sizeof(Register));
 670     Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 671         block->appendNew<Value>(proc, Add, Origin(), framePointer, offSetOfArgumentCount));
 672
 673     Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, Origin(), signature->arguments.size());
 674
 675     CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, Origin(),
 676         block->appendNew<Value>(proc, Above, Origin(), expectedArgumentCount, argumentCount));
 677     argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 678         jit.breakpoint();
 679     });
 680
 681     // Move memory values to the approriate places, if needed.
 682     Value* baseMemory = nullptr;
 683     Vector<Value*> sizes;
 684     if (memory) {
 685         baseMemory = block->appendNew<ConstPtrValue>(proc, Origin(), memory->memory());
 686         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 687             block->appendNew<ConstPtrValue>(proc, Origin(), bitwise_cast<char*>(memory) + Memory::offsetOfSize()));
 688         sizes.reserveCapacity(memory->pinnedRegisters().sizeRegisters.size());
 689         for (auto info : memory->pinnedRegisters().sizeRegisters) {
 690             sizes.append(block->appendNew<Value>(proc, Sub, Origin(), size,
 691                 block->appendNew<Const32Value>(proc, Origin(), info.sizeOffset)));
 692         }
 693     }
 694
 695     // Get our arguments.
 696     Vector<Value*> arguments;
 697     jscCallingConvention().loadArguments(signature->arguments, proc, block, Origin(), [&] (Value* argument, unsigned) {
 698         arguments.append(argument);
 699     });
 700
 701     // Move the arguments into place.
 702     Value* result = wasmCallingConvention().setupCall(proc, block, Origin(), arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
 703         if (memory) {
 704             ASSERT(sizes.size() == memory->pinnedRegisters().sizeRegisters.size());
 705             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory->pinnedRegisters().baseMemoryPointer)));
 706             for (unsigned i = 0; i < sizes.size(); ++i)
 707                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory->pinnedRegisters().sizeRegisters[i].sizeRegister)));
 708         }
 709
 710         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 711             AllowMacroScratchRegisterUsage allowScratch(jit);
 712
 713             CCallHelpers::Call call = jit.call();
 714             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 715                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
 716             });
 717         });
 718     });
 719
 720     // Return the result, if needed.
 721     switch (signature->returnType) {
 722     case Wasm::Void:
 723         block->appendNewControlValue(proc, B3::Return, Origin());
 724         break;
 725     case Wasm::F32:
 726     case Wasm::F64:
 727         result = block->appendNew<Value>(proc, BitwiseCast, Origin(), result);
 728         FALLTHROUGH;
 729     case Wasm::I32:
 730     case Wasm::I64:
 731         block->appendNewControlValue(proc, B3::Return, Origin(), result);
 732         break;
 733     case Wasm::Func:
 734     case Wasm::Anyfunc:
 735         RELEASE_ASSERT_NOT_REACHED();
 736     }
 737
 738     return std::make_unique<Compilation>(vm, proc);
 739 }
 740
 741 std::unique_ptr<WasmInternalFunction> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, Memory* memory, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const FunctionIndexSpace& functionIndexSpace, unsigned optLevel)
 742 {
 743     auto result = std::make_unique<WasmInternalFunction>();
 744
 745     Procedure procedure;
 746     B3IRGenerator context(memory, procedure, result.get(), unlinkedWasmToWasmCalls);
 747     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace);
 748     if (!parser.parse())
 749         RELEASE_ASSERT_NOT_REACHED();
 750
 751     procedure.resetReachability();
 752     validate(procedure, "After parsing:\n");
 753
 754     if (verbose)
 755         dataLog("Pre SSA: ", procedure);
 756     fixSSA(procedure);
 757     if (verbose)
 758         dataLog("Post SSA: ", procedure);
 759
 760     result->code = std::make_unique<Compilation>(vm, procedure, optLevel);
 761     result->jsToWasmEntryPoint = createJSToWasmWrapper(vm, signature, result->code->code(), memory);
 762     return result;
 763 }
 764
 765 // Custom wasm ops. These are the ones too messy to do in wasm.json.
 766
 767 template<>
 768 bool B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result)
 769 {
 770     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 771     patchpoint->append(arg, ValueRep::SomeRegister);
 772     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 773         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
 774     });
 775     patchpoint->effects = Effects::none();
 776     result = patchpoint;
 777     return true;
 778 }
 779
 780 template<>
 781 bool B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result)
 782 {
 783     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
 784     patchpoint->append(arg, ValueRep::SomeRegister);
 785     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 786         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
 787     });
 788     patchpoint->effects = Effects::none();
 789     result = patchpoint;
 790     return true;
 791 }
 792
 793 template<>
 794 bool B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result)
 795 {
 796     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 797     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 798     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
 799     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 800     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
 801     return true;
 802 }
 803
 804 template<>
 805 bool B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result)
 806 {
 807     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 808     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 809     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
 810     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 811     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
 812     return true;
 813 }
 814
 815 template<>
 816 bool B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result)
 817 {
 818     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 819     if (isX86())
 820         patchpoint->numGPScratchRegisters = 1;
 821     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
 822     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 823         AllowMacroScratchRegisterUsage allowScratch(jit);
 824 #if CPU(X86_64)
 825         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
 826 #else
 827         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
 828 #endif
 829     });
 830     patchpoint->effects = Effects::none();
 831     result = patchpoint;
 832     return true;
 833 }
 834
 835 template<>
 836 bool B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result)
 837 {
 838     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 839     if (isX86())
 840         patchpoint->numGPScratchRegisters = 1;
 841     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
 842     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 843         AllowMacroScratchRegisterUsage allowScratch(jit);
 844 #if CPU(X86_64)
 845         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
 846 #else
 847         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
 848 #endif
 849     });
 850     patchpoint->effects = Effects::none();
 851     result = patchpoint;
 852     return true;
 853 }
 854
 855 template<>
 856 bool B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result)
 857 {
 858     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 859     patchpoint->append(arg, ValueRep::SomeRegister);
 860     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 861         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
 862     });
 863     patchpoint->effects = Effects::none();
 864     result = patchpoint;
 865     return true;
 866 }
 867
 868 template<>
 869 bool B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result)
 870 {
 871     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 872     patchpoint->append(arg, ValueRep::SomeRegister);
 873     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 874         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
 875     });
 876     patchpoint->effects = Effects::none();
 877     result = patchpoint;
 878     return true;
 879 }
 880
 881 template<>
 882 bool B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result)
 883 {
 884     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 885     patchpoint->append(arg, ValueRep::SomeRegister);
 886     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 887         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
 888     });
 889     patchpoint->effects = Effects::none();
 890     result = patchpoint;
 891     return true;
 892 }
 893
 894 template<>
 895 bool B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result)
 896 {
 897     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 898     patchpoint->append(arg, ValueRep::SomeRegister);
 899     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 900         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
 901     });
 902     patchpoint->effects = Effects::none();
 903     result = patchpoint;
 904     return true;
 905 }
 906
 907 template<>
 908 bool B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result)
 909 {
 910     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
 911     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
 912     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 913         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 914         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
 915     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 916     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 917     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 918         jit.breakpoint();
 919     });
 920     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 921     patchpoint->append(arg, ValueRep::SomeRegister);
 922     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 923         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
 924     });
 925     patchpoint->effects = Effects::none();
 926     result = patchpoint;
 927     return true;
 928 }
 929
 930 template<>
 931 bool B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result)
 932 {
 933     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
 934     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
 935     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 936         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 937         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
 938     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 939     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 940     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 941         jit.breakpoint();
 942     });
 943     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 944     patchpoint->append(arg, ValueRep::SomeRegister);
 945     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 946         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
 947     });
 948     patchpoint->effects = Effects::none();
 949     result = patchpoint;
 950     return true;
 951 }
 952
 953
 954 template<>
 955 bool B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result)
 956 {
 957     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
 958     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
 959     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 960         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 961         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
 962     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 963     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 964     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 965         jit.breakpoint();
 966     });
 967     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 968     patchpoint->append(arg, ValueRep::SomeRegister);
 969     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 970         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
 971     });
 972     patchpoint->effects = Effects::none();
 973     result = patchpoint;
 974     return true;
 975 }
 976
 977 template<>
 978 bool B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result)
 979 {
 980     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
 981     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
 982     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 983         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 984         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
 985     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 986     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 987     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 988         jit.breakpoint();
 989     });
 990     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 991     patchpoint->append(arg, ValueRep::SomeRegister);
 992     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 993         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
 994     });
 995     patchpoint->effects = Effects::none();
 996     result = patchpoint;
 997     return true;
 998 }
 999
1000 template<>
1001 bool B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result)
1002 {
1003     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1004     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1005     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1006         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1007         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1008     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1009     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1010     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1011         jit.breakpoint();
1012     });
1013     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1014     patchpoint->append(arg, ValueRep::SomeRegister);
1015     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1016         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1017     });
1018     patchpoint->effects = Effects::none();
1019     result = patchpoint;
1020     return true;
1021 }
1022
1023 template<>
1024 bool B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result)
1025 {
1026     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1027     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1028     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1029         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1030         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1031     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1032     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1033     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1034         jit.breakpoint();
1035     });
1036
1037     Value* constant;
1038     if (isX86()) {
1039         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1040         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1041         // so we can pool them if needed.
1042         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1043     }
1044     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1045     patchpoint->append(arg, ValueRep::SomeRegister);
1046     if (isX86()) {
1047         patchpoint->append(constant, ValueRep::SomeRegister);
1048         patchpoint->numFPScratchRegisters = 1;
1049     }
1050     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1051         AllowMacroScratchRegisterUsage allowScratch(jit);
1052         FPRReg scratch = InvalidFPRReg;
1053         FPRReg constant = InvalidFPRReg;
1054         if (isX86()) {
1055             scratch = params.fpScratch(0);
1056             constant = params[2].fpr();
1057         }
1058         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1059     });
1060     patchpoint->effects = Effects::none();
1061     result = patchpoint;
1062     return true;
1063 }
1064
1065 template<>
1066 bool B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result)
1067 {
1068     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1069     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1070     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1071         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1072         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1073     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1074     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1075     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1076         jit.breakpoint();
1077     });
1078     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1079     patchpoint->append(arg, ValueRep::SomeRegister);
1080     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1081         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1082     });
1083     patchpoint->effects = Effects::none();
1084     result = patchpoint;
1085     return true;
1086 }
1087
1088 template<>
1089 bool B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result)
1090 {
1091     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1092     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1093     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1094         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1095         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1096     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1097     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1098     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1099         jit.breakpoint();
1100     });
1101
1102     Value* constant;
1103     if (isX86()) {
1104         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1105         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1106         // so we can pool them if needed.
1107         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1108     }
1109     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1110     patchpoint->append(arg, ValueRep::SomeRegister);
1111     if (isX86()) {
1112         patchpoint->append(constant, ValueRep::SomeRegister);
1113         patchpoint->numFPScratchRegisters = 1;
1114     }
1115     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1116         AllowMacroScratchRegisterUsage allowScratch(jit);
1117         FPRReg scratch = InvalidFPRReg;
1118         FPRReg constant = InvalidFPRReg;
1119         if (isX86()) {
1120             scratch = params.fpScratch(0);
1121             constant = params[2].fpr();
1122         }
1123         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1124     });
1125     patchpoint->effects = Effects::none();
1126     result = patchpoint;
1127     return true;
1128 }
1129
1130 } } // namespace JSC::Wasm
1131
1132 #include "WasmB3IRGeneratorInlines.h"
1133
1134 #endif // ENABLE(WEBASSEMBLY)