Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp

   1 /*
   2  * Copyright (C) 2016 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "WasmB3IRGenerator.h"
  28
  29 #if ENABLE(WEBASSEMBLY)
  30
  31 #include "B3BasicBlockInlines.h"
  32 #include "B3CCallValue.h"
  33 #include "B3ConstPtrValue.h"
  34 #include "B3FixSSA.h"
  35 #include "B3StackmapGenerationParams.h"
  36 #include "B3SwitchValue.h"
  37 #include "B3Validate.h"
  38 #include "B3ValueInlines.h"
  39 #include "B3Variable.h"
  40 #include "B3VariableValue.h"
  41 #include "B3WasmAddressValue.h"
  42 #include "B3WasmBoundsCheckValue.h"
  43 #include "JSWebAssemblyInstance.h"
  44 #include "JSWebAssemblyModule.h"
  45 #include "VirtualRegister.h"
  46 #include "WasmCallingConvention.h"
  47 #include "WasmFunctionParser.h"
  48 #include "WasmMemory.h"
  49 #include <wtf/Optional.h>
  50
  51 void dumpProcedure(void* ptr)
  52 {
  53     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
  54     proc->dump(WTF::dataFile());
  55 }
  56
  57 namespace JSC { namespace Wasm {
  58
  59 using namespace B3;
  60
  61 namespace {
  62 const bool verbose = false;
  63 }
  64
  65 class B3IRGenerator {
  66 public:
  67     struct ControlData {
  68         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
  69             : blockType(type)
  70             , continuation(continuation)
  71             , special(special)
  72         {
  73             if (signature != Void)
  74                 result.append(proc.addVariable(toB3Type(signature)));
  75         }
  76
  77         ControlData()
  78         {
  79         }
  80
  81         void dump(PrintStream& out) const
  82         {
  83             switch (type()) {
  84             case BlockType::If:
  85                 out.print("If:    ");
  86                 break;
  87             case BlockType::Block:
  88                 out.print("Block: ");
  89                 break;
  90             case BlockType::Loop:
  91                 out.print("Loop:  ");
  92                 break;
  93             }
  94             out.print("Continuation: ", *continuation, ", Special: ");
  95             if (special)
  96                 out.print(*special);
  97             else
  98                 out.print("None");
  99         }
 100
 101         BlockType type() const { return blockType; }
 102
 103         bool hasNonVoidSignature() const { return result.size(); }
 104
 105         BasicBlock* targetBlockForBranch()
 106         {
 107             if (type() == BlockType::Loop)
 108                 return special;
 109             return continuation;
 110         }
 111
 112         void convertIfToBlock()
 113         {
 114             ASSERT(type() == BlockType::If);
 115             blockType = BlockType::Block;
 116             special = nullptr;
 117         }
 118
 119     private:
 120         friend class B3IRGenerator;
 121         BlockType blockType;
 122         BasicBlock* continuation;
 123         BasicBlock* special;
 124         Vector<Variable*, 1> result;
 125     };
 126
 127     typedef Value* ExpressionType;
 128     typedef ControlData ControlType;
 129     typedef Vector<ExpressionType, 1> ExpressionList;
 130     typedef Vector<Variable*, 1> ResultList;
 131     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
 132
 133     static constexpr ExpressionType emptyExpression = nullptr;
 134
 135     B3IRGenerator(const MemoryInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, const ImmutableFunctionIndexSpace&);
 136
 137     bool WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
 138     bool WARN_UNUSED_RETURN addLocal(Type, uint32_t);
 139     ExpressionType addConstant(Type, uint64_t);
 140
 141     // Locals
 142     bool WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
 143     bool WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
 144
 145     // Memory
 146     bool WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
 147     bool WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
 148
 149     // Basic operators
 150     template<OpType>
 151     bool WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
 152     template<OpType>
 153     bool WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
 154     bool WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
 155
 156     // Control flow
 157     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
 158     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
 159     bool WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
 160     bool WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
 161     bool WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
 162
 163     bool WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
 164     bool WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
 165     bool WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
 166     bool WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
 167     bool WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
 168
 169     // Calls
 170     bool WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 171     bool WARN_UNUSED_RETURN addCallIndirect(const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 172
 173     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
 174
 175     void setErrorMessage(String&&) { UNREACHABLE_FOR_PLATFORM(); }
 176
 177 private:
 178     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
 179     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
 180     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
 181
 182     void unify(Variable* target, const ExpressionType source);
 183     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
 184     Value* zeroForType(Type);
 185
 186     const ImmutableFunctionIndexSpace& m_functionIndexSpace;
 187     Procedure& m_proc;
 188     BasicBlock* m_currentBlock;
 189     Vector<Variable*> m_locals;
 190     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
 191     GPRReg m_memoryBaseGPR;
 192     GPRReg m_memorySizeGPR;
 193     Value* m_zeroValues[numTypes];
 194     Value* m_functionIndexSpaceValue;
 195 };
 196
 197 B3IRGenerator::B3IRGenerator(const MemoryInformation& memory, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace)
 198     : m_functionIndexSpace(functionIndexSpace)
 199     , m_proc(procedure)
 200     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
 201 {
 202     m_currentBlock = m_proc.addBlock();
 203
 204     for (unsigned i = 0; i < numTypes; ++i) {
 205         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
 206         case B3::Int32:
 207         case B3::Int64:
 208         case B3::Float:
 209         case B3::Double:
 210             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
 211             break;
 212         case B3::Void:
 213             m_zeroValues[i] = nullptr;
 214             break;
 215         }
 216     }
 217
 218     if (!!memory) {
 219         m_memoryBaseGPR = memory.pinnedRegisters().baseMemoryPointer;
 220         m_proc.pinRegister(m_memoryBaseGPR);
 221         ASSERT(!memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
 222         m_memorySizeGPR = memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
 223         for (const PinnedSizeRegisterInfo& info : memory.pinnedRegisters().sizeRegisters)
 224             m_proc.pinRegister(info.sizeRegister);
 225
 226         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
 227             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
 228             // FIXME: This should unwind the stack and throw a JS exception. See: https://bugs.webkit.org/show_bug.cgi?id=163351
 229             jit.breakpoint();
 230         });
 231     }
 232
 233     wasmCallingConvention().setupFrameInPrologue(compilation, m_proc, Origin(), m_currentBlock);
 234
 235     m_functionIndexSpaceValue = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), functionIndexSpace.buffer.get());
 236 }
 237
 238 Value* B3IRGenerator::zeroForType(Type type)
 239 {
 240     ASSERT(type != Void);
 241     Value* zeroValue = m_zeroValues[linearizeType(type)];
 242     ASSERT(zeroValue);
 243     return zeroValue;
 244 }
 245
 246 bool B3IRGenerator::addLocal(Type type, uint32_t count)
 247 {
 248     if (!m_locals.tryReserveCapacity(m_locals.size() + count))
 249         return false;
 250
 251     for (uint32_t i = 0; i < count; ++i) {
 252         Variable* local = m_proc.addVariable(toB3Type(type));
 253         m_locals.uncheckedAppend(local);
 254         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
 255     }
 256     return true;
 257 }
 258
 259 bool B3IRGenerator::addArguments(const Vector<Type>& types)
 260 {
 261     ASSERT(!m_locals.size());
 262     if (!m_locals.tryReserveCapacity(types.size()))
 263         return false;
 264
 265     m_locals.grow(types.size());
 266     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
 267         [&] (ExpressionType argument, unsigned i) {
 268             Variable* argumentVariable = m_proc.addVariable(argument->type());
 269             m_locals[i] = argumentVariable;
 270             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
 271         });
 272     return true;
 273 }
 274
 275 bool B3IRGenerator::getLocal(uint32_t index, ExpressionType& result)
 276 {
 277     ASSERT(m_locals[index]);
 278     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
 279     return true;
 280 }
 281
 282 bool B3IRGenerator::setLocal(uint32_t index, ExpressionType value)
 283 {
 284     ASSERT(m_locals[index]);
 285     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
 286     return true;
 287 }
 288
 289 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
 290 {
 291     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
 292     ASSERT(sizeOfOperation + offset > offset);
 293     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
 294     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
 295     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
 296 }
 297
 298 inline uint32_t sizeOfLoadOp(LoadOpType op)
 299 {
 300     switch (op) {
 301     case LoadOpType::I32Load8S:
 302     case LoadOpType::I32Load8U:
 303     case LoadOpType::I64Load8S:
 304     case LoadOpType::I64Load8U:
 305         return 1;
 306     case LoadOpType::I32Load16S:
 307     case LoadOpType::I64Load16S:
 308         return 2;
 309     case LoadOpType::I32Load:
 310     case LoadOpType::I64Load32S:
 311     case LoadOpType::I64Load32U:
 312     case LoadOpType::F32Load:
 313         return 4;
 314     case LoadOpType::I64Load:
 315     case LoadOpType::F64Load:
 316         return 8;
 317     case LoadOpType::I32Load16U:
 318     case LoadOpType::I64Load16U:
 319         break;
 320     }
 321     RELEASE_ASSERT_NOT_REACHED();
 322 }
 323
 324 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
 325 {
 326     switch (op) {
 327     case LoadOpType::I32Load8S: {
 328         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 329     }
 330
 331     case LoadOpType::I64Load8S: {
 332         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 333         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 334     }
 335
 336     case LoadOpType::I32Load8U: {
 337         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 338     }
 339
 340     case LoadOpType::I64Load8U: {
 341         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 342         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 343     }
 344
 345     case LoadOpType::I32Load16S: {
 346         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 347     }
 348     case LoadOpType::I64Load16S: {
 349         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 350         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 351     }
 352
 353     case LoadOpType::I32Load: {
 354         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 355     }
 356
 357     case LoadOpType::I64Load32U: {
 358         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 359         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 360     }
 361
 362     case LoadOpType::I64Load32S: {
 363         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 364         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 365     }
 366
 367     case LoadOpType::I64Load: {
 368         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
 369     }
 370
 371     case LoadOpType::F32Load: {
 372         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
 373     }
 374
 375     case LoadOpType::F64Load: {
 376         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
 377     }
 378
 379     // B3 doesn't support Load16Z yet.
 380     case LoadOpType::I32Load16U:
 381     case LoadOpType::I64Load16U:
 382         break;
 383     }
 384     RELEASE_ASSERT_NOT_REACHED();
 385 }
 386
 387 bool B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset)
 388 {
 389     ASSERT(pointer->type() == Int32);
 390
 391     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
 392     return true;
 393 }
 394
 395 inline uint32_t sizeOfStoreOp(StoreOpType op)
 396 {
 397     switch (op) {
 398     case StoreOpType::I32Store8:
 399     case StoreOpType::I64Store8:
 400         return 1;
 401     case StoreOpType::I32Store16:
 402     case StoreOpType::I64Store16:
 403         return 2;
 404     case StoreOpType::I32Store:
 405     case StoreOpType::I64Store32:
 406     case StoreOpType::F32Store:
 407         return 4;
 408     case StoreOpType::I64Store:
 409     case StoreOpType::F64Store:
 410         return 8;
 411     }
 412     RELEASE_ASSERT_NOT_REACHED();
 413 }
 414
 415
 416 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
 417 {
 418     switch (op) {
 419     case StoreOpType::I64Store8:
 420         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 421         FALLTHROUGH;
 422
 423     case StoreOpType::I32Store8:
 424         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
 425         return;
 426
 427     case StoreOpType::I64Store16:
 428         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 429         FALLTHROUGH;
 430
 431     case StoreOpType::I32Store16:
 432         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
 433         return;
 434
 435     case StoreOpType::I64Store32:
 436         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 437         FALLTHROUGH;
 438
 439     case StoreOpType::I64Store:
 440     case StoreOpType::I32Store:
 441     case StoreOpType::F32Store:
 442     case StoreOpType::F64Store:
 443         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
 444         return;
 445     }
 446     RELEASE_ASSERT_NOT_REACHED();
 447 }
 448
 449 bool B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset)
 450 {
 451     ASSERT(pointer->type() == Int32);
 452
 453     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
 454     return true;
 455 }
 456
 457 bool B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result)
 458 {
 459     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
 460     return true;
 461 }
 462
 463 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
 464 {
 465     switch (type) {
 466     case Wasm::I32:
 467         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
 468     case Wasm::I64:
 469         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
 470     case Wasm::F32:
 471         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
 472     case Wasm::F64:
 473         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
 474     case Wasm::Void:
 475     case Wasm::Func:
 476     case Wasm::Anyfunc:
 477         break;
 478     }
 479     RELEASE_ASSERT_NOT_REACHED();
 480     return nullptr;
 481 }
 482
 483 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
 484 {
 485     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
 486 }
 487
 488 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
 489 {
 490     BasicBlock* body = m_proc.addBlock();
 491     BasicBlock* continuation = m_proc.addBlock();
 492     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
 493     body->addPredecessor(m_currentBlock);
 494     m_currentBlock = body;
 495     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
 496 }
 497
 498 bool B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result)
 499 {
 500     // FIXME: This needs to do some kind of stack passing.
 501
 502     BasicBlock* taken = m_proc.addBlock();
 503     BasicBlock* notTaken = m_proc.addBlock();
 504     BasicBlock* continuation = m_proc.addBlock();
 505
 506     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 507     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
 508     taken->addPredecessor(m_currentBlock);
 509     notTaken->addPredecessor(m_currentBlock);
 510
 511     m_currentBlock = taken;
 512     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
 513     return true;
 514 }
 515
 516 bool B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack)
 517 {
 518     unifyValuesWithBlock(currentStack, data.result);
 519     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 520     return addElseToUnreachable(data);
 521 }
 522
 523 bool B3IRGenerator::addElseToUnreachable(ControlData& data)
 524 {
 525     ASSERT(data.type() == BlockType::If);
 526     m_currentBlock = data.special;
 527     data.convertIfToBlock();
 528     return true;
 529 }
 530
 531 bool B3IRGenerator::addReturn(const ExpressionList& returnValues)
 532 {
 533     ASSERT(returnValues.size() <= 1);
 534     if (returnValues.size())
 535         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
 536     else
 537         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
 538     return true;
 539 }
 540
 541 bool B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues)
 542 {
 543     if (data.type() != BlockType::Loop)
 544         unifyValuesWithBlock(returnValues, data.result);
 545
 546     BasicBlock* target = data.targetBlockForBranch();
 547     if (condition) {
 548         BasicBlock* continuation = m_proc.addBlock();
 549         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 550         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
 551         target->addPredecessor(m_currentBlock);
 552         continuation->addPredecessor(m_currentBlock);
 553         m_currentBlock = continuation;
 554     } else {
 555         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
 556         target->addPredecessor(m_currentBlock);
 557     }
 558
 559     return true;
 560 }
 561
 562 bool B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack)
 563 {
 564     for (size_t i = 0; i < targets.size(); ++i)
 565         unifyValuesWithBlock(expressionStack, targets[i]->result);
 566     unifyValuesWithBlock(expressionStack, defaultTarget.result);
 567
 568     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
 569     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
 570     for (size_t i = 0; i < targets.size(); ++i)
 571         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
 572
 573     return true;
 574 }
 575
 576 bool B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack)
 577 {
 578     ControlData& data = entry.controlData;
 579
 580     unifyValuesWithBlock(expressionStack, data.result);
 581     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 582     data.continuation->addPredecessor(m_currentBlock);
 583
 584     return addEndToUnreachable(entry);
 585 }
 586
 587
 588 bool B3IRGenerator::addEndToUnreachable(ControlEntry& entry)
 589 {
 590     ControlData& data = entry.controlData;
 591     m_currentBlock = data.continuation;
 592
 593     if (data.type() == BlockType::If) {
 594         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
 595         m_currentBlock->addPredecessor(data.special);
 596     }
 597
 598     for (Variable* result : data.result)
 599         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
 600
 601     return true;
 602 }
 603
 604 bool B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result)
 605 {
 606     ASSERT(signature->arguments.size() == args.size());
 607
 608     Type returnType = signature->returnType;
 609
 610     size_t callIndex = m_unlinkedWasmToWasmCalls.size();
 611     m_unlinkedWasmToWasmCalls.grow(callIndex + 1);
 612     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
 613         [&] (PatchpointValue* patchpoint) {
 614             patchpoint->effects.writesPinned = true;
 615             patchpoint->effects.readsPinned = true;
 616
 617             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 618                 AllowMacroScratchRegisterUsage allowScratch(jit);
 619
 620                 CCallHelpers::Call call = jit.call();
 621
 622                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 623                     m_unlinkedWasmToWasmCalls[callIndex] = { linkBuffer.locationOf(call), functionIndex };
 624                 });
 625             });
 626         });
 627     return true;
 628 }
 629
 630 bool B3IRGenerator::addCallIndirect(const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result)
 631 {
 632     ExpressionType calleeIndex = args.takeLast();
 633     ASSERT(signature->arguments.size() == args.size());
 634
 635     // Check the index we are looking for is valid.
 636     {
 637         ExpressionType maxValidIndex = m_currentBlock->appendIntConstant(m_proc, Origin(), Int32, m_functionIndexSpace.size);
 638         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 639             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), m_zeroValues[linearizeType(I32)],
 640                 m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), calleeIndex, maxValidIndex)));
 641
 642         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 643             jit.breakpoint();
 644         });
 645     }
 646
 647     // Compute the offset in the function index space we are looking for.
 648     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
 649         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
 650         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
 651     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), m_functionIndexSpaceValue, offset);
 652
 653     // Check the signature matches the value we expect.
 654     {
 655         ExpressionType calleeSignature = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signature));
 656         ExpressionType expectedSignature = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), signature);
 657         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 658             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignature, expectedSignature));
 659
 660         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 661             jit.breakpoint();
 662         });
 663     }
 664
 665     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
 666
 667     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(signature->returnType),
 668         [&] (PatchpointValue* patchpoint) {
 669             patchpoint->effects.writesPinned = true;
 670             patchpoint->effects.readsPinned = true;
 671
 672             patchpoint->append(calleeCode, ValueRep::SomeRegister);
 673
 674             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
 675                 jit.call(params[0].gpr());
 676             });
 677         });
 678     return true;
 679 }
 680
 681 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
 682 {
 683     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
 684 }
 685
 686 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
 687 {
 688     ASSERT(result.size() <= resultStack.size());
 689
 690     for (size_t i = 0; i < result.size(); ++i)
 691         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
 692 }
 693
 694 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
 695 {
 696     dataLogLn(comma, "ExpressionStack:");
 697     for (const auto& expression : expressionStack)
 698         dataLogLn(comma, *expression);
 699 }
 700
 701 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
 702 {
 703     dataLogLn("Processing Graph:");
 704     dataLog(m_proc);
 705     dataLogLn("With current block:", *m_currentBlock);
 706     dataLogLn("Control stack:");
 707     for (auto& data : controlStack) {
 708         dataLogLn("  ", data.controlData);
 709         if (data.enclosedExpressionStack.size()) {
 710             CommaPrinter comma("    ", "  with ");
 711             dumpExpressionStack(comma, data.enclosedExpressionStack);
 712         }
 713     }
 714
 715     CommaPrinter comma("  ", "");
 716     dumpExpressionStack(comma, expressionStack);
 717     dataLogLn("\n");
 718 }
 719
 720 static std::unique_ptr<Compilation> createJSToWasmWrapper(VM& vm, const Signature* signature, MacroAssemblerCodePtr mainFunction, const MemoryInformation& memory)
 721 {
 722     Procedure proc;
 723     BasicBlock* block = proc.addBlock();
 724
 725     // Check argument count is sane.
 726     Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, Origin());
 727     Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, Origin(), CallFrameSlot::argumentCount * sizeof(Register));
 728     Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 729         block->appendNew<Value>(proc, Add, Origin(), framePointer, offSetOfArgumentCount));
 730
 731     Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, Origin(), signature->arguments.size());
 732
 733     CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, Origin(),
 734         block->appendNew<Value>(proc, Above, Origin(), expectedArgumentCount, argumentCount));
 735     argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 736         jit.breakpoint();
 737     });
 738
 739     // Move memory values to the approriate places, if needed.
 740     Value* baseMemory = nullptr;
 741     Vector<Value*> sizes;
 742     if (!!memory) {
 743         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
 744             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
 745         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 746             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
 747         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
 748         for (auto info : memory.pinnedRegisters().sizeRegisters) {
 749             sizes.append(block->appendNew<Value>(proc, Sub, Origin(), size,
 750                 block->appendNew<Const32Value>(proc, Origin(), info.sizeOffset)));
 751         }
 752     }
 753
 754     // Get our arguments.
 755     Vector<Value*> arguments;
 756     jscCallingConvention().loadArguments(signature->arguments, proc, block, Origin(), [&] (Value* argument, unsigned) {
 757         arguments.append(argument);
 758     });
 759
 760     // Move the arguments into place.
 761     Value* result = wasmCallingConvention().setupCall(proc, block, Origin(), arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
 762         if (!!memory) {
 763             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
 764             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
 765             for (unsigned i = 0; i < sizes.size(); ++i)
 766                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
 767         }
 768
 769         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 770             AllowMacroScratchRegisterUsage allowScratch(jit);
 771
 772             CCallHelpers::Call call = jit.call();
 773             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 774                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
 775             });
 776         });
 777     });
 778
 779     // Return the result, if needed.
 780     switch (signature->returnType) {
 781     case Wasm::Void:
 782         block->appendNewControlValue(proc, B3::Return, Origin());
 783         break;
 784     case Wasm::F32:
 785     case Wasm::F64:
 786         result = block->appendNew<Value>(proc, BitwiseCast, Origin(), result);
 787         FALLTHROUGH;
 788     case Wasm::I32:
 789     case Wasm::I64:
 790         block->appendNewControlValue(proc, B3::Return, Origin(), result);
 791         break;
 792     case Wasm::Func:
 793     case Wasm::Anyfunc:
 794         RELEASE_ASSERT_NOT_REACHED();
 795     }
 796
 797     return std::make_unique<Compilation>(vm, proc);
 798 }
 799
 800 std::unique_ptr<WasmInternalFunction> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace, const ModuleInformation& info, unsigned optLevel)
 801 {
 802     auto result = std::make_unique<WasmInternalFunction>();
 803
 804     Procedure procedure;
 805     B3IRGenerator context(info.memory, procedure, result.get(), unlinkedWasmToWasmCalls, functionIndexSpace);
 806     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace, info);
 807     if (!parser.parse())
 808         RELEASE_ASSERT_NOT_REACHED();
 809
 810     procedure.resetReachability();
 811     validate(procedure, "After parsing:\n");
 812
 813     if (verbose)
 814         dataLog("Pre SSA: ", procedure);
 815     fixSSA(procedure);
 816     if (verbose)
 817         dataLog("Post SSA: ", procedure);
 818
 819     result->code = std::make_unique<Compilation>(vm, procedure, optLevel);
 820     result->jsToWasmEntryPoint = createJSToWasmWrapper(vm, signature, result->code->code(), info.memory);
 821     return result;
 822 }
 823
 824 // Custom wasm ops. These are the ones too messy to do in wasm.json.
 825
 826 template<>
 827 bool B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result)
 828 {
 829     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 830     patchpoint->append(arg, ValueRep::SomeRegister);
 831     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 832         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
 833     });
 834     patchpoint->effects = Effects::none();
 835     result = patchpoint;
 836     return true;
 837 }
 838
 839 template<>
 840 bool B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result)
 841 {
 842     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
 843     patchpoint->append(arg, ValueRep::SomeRegister);
 844     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 845         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
 846     });
 847     patchpoint->effects = Effects::none();
 848     result = patchpoint;
 849     return true;
 850 }
 851
 852 template<>
 853 bool B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result)
 854 {
 855     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 856     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 857     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
 858     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 859     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
 860     return true;
 861 }
 862
 863 template<>
 864 bool B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result)
 865 {
 866     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 867     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 868     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
 869     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 870     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
 871     return true;
 872 }
 873
 874 template<>
 875 bool B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result)
 876 {
 877     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 878     if (isX86())
 879         patchpoint->numGPScratchRegisters = 1;
 880     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
 881     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 882         AllowMacroScratchRegisterUsage allowScratch(jit);
 883 #if CPU(X86_64)
 884         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
 885 #else
 886         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
 887 #endif
 888     });
 889     patchpoint->effects = Effects::none();
 890     result = patchpoint;
 891     return true;
 892 }
 893
 894 template<>
 895 bool B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result)
 896 {
 897     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 898     if (isX86())
 899         patchpoint->numGPScratchRegisters = 1;
 900     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
 901     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 902         AllowMacroScratchRegisterUsage allowScratch(jit);
 903 #if CPU(X86_64)
 904         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
 905 #else
 906         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
 907 #endif
 908     });
 909     patchpoint->effects = Effects::none();
 910     result = patchpoint;
 911     return true;
 912 }
 913
 914 template<>
 915 bool B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result)
 916 {
 917     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 918     patchpoint->append(arg, ValueRep::SomeRegister);
 919     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 920         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
 921     });
 922     patchpoint->effects = Effects::none();
 923     result = patchpoint;
 924     return true;
 925 }
 926
 927 template<>
 928 bool B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result)
 929 {
 930     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 931     patchpoint->append(arg, ValueRep::SomeRegister);
 932     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 933         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
 934     });
 935     patchpoint->effects = Effects::none();
 936     result = patchpoint;
 937     return true;
 938 }
 939
 940 template<>
 941 bool B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result)
 942 {
 943     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 944     patchpoint->append(arg, ValueRep::SomeRegister);
 945     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 946         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
 947     });
 948     patchpoint->effects = Effects::none();
 949     result = patchpoint;
 950     return true;
 951 }
 952
 953 template<>
 954 bool B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result)
 955 {
 956     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
 957     patchpoint->append(arg, ValueRep::SomeRegister);
 958     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 959         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
 960     });
 961     patchpoint->effects = Effects::none();
 962     result = patchpoint;
 963     return true;
 964 }
 965
 966 template<>
 967 bool B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result)
 968 {
 969     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
 970     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
 971     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 972         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 973         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
 974     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 975     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 976     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 977         jit.breakpoint();
 978     });
 979     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 980     patchpoint->append(arg, ValueRep::SomeRegister);
 981     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 982         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
 983     });
 984     patchpoint->effects = Effects::none();
 985     result = patchpoint;
 986     return true;
 987 }
 988
 989 template<>
 990 bool B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result)
 991 {
 992     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
 993     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
 994     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
 995         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
 996         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
 997     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
 998     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
 999     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1000         jit.breakpoint();
1001     });
1002     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1003     patchpoint->append(arg, ValueRep::SomeRegister);
1004     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1005         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1006     });
1007     patchpoint->effects = Effects::none();
1008     result = patchpoint;
1009     return true;
1010 }
1011
1012
1013 template<>
1014 bool B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result)
1015 {
1016     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1017     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1018     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1019         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1020         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1021     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1022     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1023     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1024         jit.breakpoint();
1025     });
1026     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1027     patchpoint->append(arg, ValueRep::SomeRegister);
1028     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1029         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1030     });
1031     patchpoint->effects = Effects::none();
1032     result = patchpoint;
1033     return true;
1034 }
1035
1036 template<>
1037 bool B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result)
1038 {
1039     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1040     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1041     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1042         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1043         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1044     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1045     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1046     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1047         jit.breakpoint();
1048     });
1049     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1050     patchpoint->append(arg, ValueRep::SomeRegister);
1051     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1052         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1053     });
1054     patchpoint->effects = Effects::none();
1055     result = patchpoint;
1056     return true;
1057 }
1058
1059 template<>
1060 bool B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result)
1061 {
1062     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1063     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1064     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1065         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1066         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1067     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1068     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1069     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1070         jit.breakpoint();
1071     });
1072     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1073     patchpoint->append(arg, ValueRep::SomeRegister);
1074     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1075         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1076     });
1077     patchpoint->effects = Effects::none();
1078     result = patchpoint;
1079     return true;
1080 }
1081
1082 template<>
1083 bool B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result)
1084 {
1085     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1086     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1087     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1088         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1089         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1090     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1091     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1092     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1093         jit.breakpoint();
1094     });
1095
1096     Value* constant;
1097     if (isX86()) {
1098         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1099         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1100         // so we can pool them if needed.
1101         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1102     }
1103     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1104     patchpoint->append(arg, ValueRep::SomeRegister);
1105     if (isX86()) {
1106         patchpoint->append(constant, ValueRep::SomeRegister);
1107         patchpoint->numFPScratchRegisters = 1;
1108     }
1109     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1110         AllowMacroScratchRegisterUsage allowScratch(jit);
1111         FPRReg scratch = InvalidFPRReg;
1112         FPRReg constant = InvalidFPRReg;
1113         if (isX86()) {
1114             scratch = params.fpScratch(0);
1115             constant = params[2].fpr();
1116         }
1117         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1118     });
1119     patchpoint->effects = Effects::none();
1120     result = patchpoint;
1121     return true;
1122 }
1123
1124 template<>
1125 bool B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result)
1126 {
1127     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1128     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1129     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1130         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1131         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1132     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1133     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1134     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1135         jit.breakpoint();
1136     });
1137     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1138     patchpoint->append(arg, ValueRep::SomeRegister);
1139     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1140         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1141     });
1142     patchpoint->effects = Effects::none();
1143     result = patchpoint;
1144     return true;
1145 }
1146
1147 template<>
1148 bool B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result)
1149 {
1150     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1151     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1152     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1153         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1154         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1155     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1156     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1157     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1158         jit.breakpoint();
1159     });
1160
1161     Value* constant;
1162     if (isX86()) {
1163         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1164         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1165         // so we can pool them if needed.
1166         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1167     }
1168     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1169     patchpoint->append(arg, ValueRep::SomeRegister);
1170     if (isX86()) {
1171         patchpoint->append(constant, ValueRep::SomeRegister);
1172         patchpoint->numFPScratchRegisters = 1;
1173     }
1174     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1175         AllowMacroScratchRegisterUsage allowScratch(jit);
1176         FPRReg scratch = InvalidFPRReg;
1177         FPRReg constant = InvalidFPRReg;
1178         if (isX86()) {
1179             scratch = params.fpScratch(0);
1180             constant = params[2].fpr();
1181         }
1182         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1183     });
1184     patchpoint->effects = Effects::none();
1185     result = patchpoint;
1186     return true;
1187 }
1188
1189 } } // namespace JSC::Wasm
1190
1191 #include "WasmB3IRGeneratorInlines.h"
1192
1193 #endif // ENABLE(WEBASSEMBLY)