Source/JavaScriptCore/wasm/WasmB3IRGenerator.cpp

   1 /*
   2  * Copyright (C) 2016 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
  14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
  17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
  21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "WasmB3IRGenerator.h"
  28
  29 #if ENABLE(WEBASSEMBLY)
  30
  31 #include "B3BasicBlockInlines.h"
  32 #include "B3CCallValue.h"
  33 #include "B3ConstPtrValue.h"
  34 #include "B3FixSSA.h"
  35 #include "B3StackmapGenerationParams.h"
  36 #include "B3SwitchValue.h"
  37 #include "B3Validate.h"
  38 #include "B3ValueInlines.h"
  39 #include "B3Variable.h"
  40 #include "B3VariableValue.h"
  41 #include "B3WasmAddressValue.h"
  42 #include "B3WasmBoundsCheckValue.h"
  43 #include "ExceptionScope.h"
  44 #include "FrameTracers.h"
  45 #include "JITExceptions.h"
  46 #include "JSCInlines.h"
  47 #include "JSWebAssemblyInstance.h"
  48 #include "JSWebAssemblyModule.h"
  49 #include "JSWebAssemblyRuntimeError.h"
  50 #include "VirtualRegister.h"
  51 #include "WasmCallingConvention.h"
  52 #include "WasmFunctionParser.h"
  53 #include "WasmMemory.h"
  54 #include <wtf/Optional.h>
  55
  56 void dumpProcedure(void* ptr)
  57 {
  58     JSC::B3::Procedure* proc = static_cast<JSC::B3::Procedure*>(ptr);
  59     proc->dump(WTF::dataFile());
  60 }
  61
  62 namespace JSC { namespace Wasm {
  63
  64 using namespace B3;
  65
  66 namespace {
  67 const bool verbose = false;
  68 }
  69
  70 class B3IRGenerator {
  71 public:
  72     struct ControlData {
  73         ControlData(Procedure& proc, Type signature, BlockType type, BasicBlock* continuation, BasicBlock* special = nullptr)
  74             : blockType(type)
  75             , continuation(continuation)
  76             , special(special)
  77         {
  78             if (signature != Void)
  79                 result.append(proc.addVariable(toB3Type(signature)));
  80         }
  81
  82         ControlData()
  83         {
  84         }
  85
  86         void dump(PrintStream& out) const
  87         {
  88             switch (type()) {
  89             case BlockType::If:
  90                 out.print("If:    ");
  91                 break;
  92             case BlockType::Block:
  93                 out.print("Block: ");
  94                 break;
  95             case BlockType::Loop:
  96                 out.print("Loop:  ");
  97                 break;
  98             }
  99             out.print("Continuation: ", *continuation, ", Special: ");
 100             if (special)
 101                 out.print(*special);
 102             else
 103                 out.print("None");
 104         }
 105
 106         BlockType type() const { return blockType; }
 107
 108         bool hasNonVoidSignature() const { return result.size(); }
 109
 110         BasicBlock* targetBlockForBranch()
 111         {
 112             if (type() == BlockType::Loop)
 113                 return special;
 114             return continuation;
 115         }
 116
 117         void convertIfToBlock()
 118         {
 119             ASSERT(type() == BlockType::If);
 120             blockType = BlockType::Block;
 121             special = nullptr;
 122         }
 123
 124     private:
 125         friend class B3IRGenerator;
 126         BlockType blockType;
 127         BasicBlock* continuation;
 128         BasicBlock* special;
 129         Vector<Variable*, 1> result;
 130     };
 131
 132     typedef Value* ExpressionType;
 133     typedef ControlData ControlType;
 134     typedef Vector<ExpressionType, 1> ExpressionList;
 135     typedef Vector<Variable*, 1> ResultList;
 136     typedef FunctionParser<B3IRGenerator>::ControlEntry ControlEntry;
 137
 138     static constexpr ExpressionType emptyExpression = nullptr;
 139
 140     typedef String ErrorType;
 141     typedef UnexpectedType<ErrorType> UnexpectedResult;
 142     typedef Expected<std::unique_ptr<WasmInternalFunction>, ErrorType> Result;
 143     typedef Expected<void, ErrorType> PartialResult;
 144     template <typename ...Args>
 145     NEVER_INLINE UnexpectedResult WARN_UNUSED_RETURN fail(Args... args) const
 146     {
 147         using namespace FailureHelper; // See ADL comment in WasmParser.h.
 148         return UnexpectedResult(makeString(ASCIILiteral("WebAssembly.Module failed compiling: "), makeString(args)...));
 149     }
 150 #define WASM_COMPILE_FAIL_IF(condition, ...) do { \
 151         if (UNLIKELY(condition))                  \
 152             return fail(__VA_ARGS__);             \
 153     } while (0)
 154
 155     B3IRGenerator(VM&, const ModuleInformation&, Procedure&, WasmInternalFunction*, Vector<UnlinkedWasmToWasmCall>&, const ImmutableFunctionIndexSpace&);
 156
 157     PartialResult WARN_UNUSED_RETURN addArguments(const Vector<Type>&);
 158     PartialResult WARN_UNUSED_RETURN addLocal(Type, uint32_t);
 159     ExpressionType addConstant(Type, uint64_t);
 160
 161     // Locals
 162     PartialResult WARN_UNUSED_RETURN getLocal(uint32_t index, ExpressionType& result);
 163     PartialResult WARN_UNUSED_RETURN setLocal(uint32_t index, ExpressionType value);
 164
 165     // Globals
 166     PartialResult WARN_UNUSED_RETURN getGlobal(uint32_t index, ExpressionType& result);
 167     PartialResult WARN_UNUSED_RETURN setGlobal(uint32_t index, ExpressionType value);
 168
 169     // Memory
 170     PartialResult WARN_UNUSED_RETURN load(LoadOpType, ExpressionType pointer, ExpressionType& result, uint32_t offset);
 171     PartialResult WARN_UNUSED_RETURN store(StoreOpType, ExpressionType pointer, ExpressionType value, uint32_t offset);
 172
 173     // Basic operators
 174     template<OpType>
 175     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType arg, ExpressionType& result);
 176     template<OpType>
 177     PartialResult WARN_UNUSED_RETURN addOp(ExpressionType left, ExpressionType right, ExpressionType& result);
 178     PartialResult WARN_UNUSED_RETURN addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result);
 179
 180     // Control flow
 181     ControlData WARN_UNUSED_RETURN addBlock(Type signature);
 182     ControlData WARN_UNUSED_RETURN addLoop(Type signature);
 183     PartialResult WARN_UNUSED_RETURN addIf(ExpressionType condition, Type signature, ControlData& result);
 184     PartialResult WARN_UNUSED_RETURN addElse(ControlData&, const ExpressionList&);
 185     PartialResult WARN_UNUSED_RETURN addElseToUnreachable(ControlData&);
 186
 187     PartialResult WARN_UNUSED_RETURN addReturn(const ExpressionList& returnValues);
 188     PartialResult WARN_UNUSED_RETURN addBranch(ControlData&, ExpressionType condition, const ExpressionList& returnValues);
 189     PartialResult WARN_UNUSED_RETURN addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTargets, const ExpressionList& expressionStack);
 190     PartialResult WARN_UNUSED_RETURN endBlock(ControlEntry&, ExpressionList& expressionStack);
 191     PartialResult WARN_UNUSED_RETURN addEndToUnreachable(ControlEntry&);
 192
 193     // Calls
 194     PartialResult WARN_UNUSED_RETURN addCall(uint32_t calleeIndex, const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 195     PartialResult WARN_UNUSED_RETURN addCallIndirect(const Signature*, Vector<ExpressionType>& args, ExpressionType& result);
 196     PartialResult WARN_UNUSED_RETURN addUnreachable();
 197
 198     void dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack);
 199
 200 private:
 201     ExpressionType emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOp);
 202     ExpressionType emitLoadOp(LoadOpType, Origin, ExpressionType pointer, uint32_t offset);
 203     void emitStoreOp(StoreOpType, Origin, ExpressionType pointer, ExpressionType value, uint32_t offset);
 204
 205     void unify(Variable* target, const ExpressionType source);
 206     void unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& stack);
 207     Value* zeroForType(Type);
 208
 209     VM& m_vm;
 210     const ImmutableFunctionIndexSpace& m_functionIndexSpace;
 211     const ModuleInformation& m_info;
 212     Procedure& m_proc;
 213     BasicBlock* m_currentBlock;
 214     Vector<Variable*> m_locals;
 215     Vector<UnlinkedWasmToWasmCall>& m_unlinkedWasmToWasmCalls; // List each call site and the function index whose address it should be patched with.
 216     GPRReg m_memoryBaseGPR;
 217     GPRReg m_memorySizeGPR;
 218     Value* m_zeroValues[numTypes];
 219     Value* m_instanceValue;
 220
 221 };
 222
 223 B3IRGenerator::B3IRGenerator(VM& vm, const ModuleInformation& info, Procedure& procedure, WasmInternalFunction* compilation, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace)
 224     : m_vm(vm)
 225     , m_functionIndexSpace(functionIndexSpace)
 226     , m_info(info)
 227     , m_proc(procedure)
 228     , m_unlinkedWasmToWasmCalls(unlinkedWasmToWasmCalls)
 229 {
 230     m_currentBlock = m_proc.addBlock();
 231
 232     for (unsigned i = 0; i < numTypes; ++i) {
 233         switch (B3::Type b3Type = toB3Type(linearizedToType(i))) {
 234         case B3::Int32:
 235         case B3::Int64:
 236         case B3::Float:
 237         case B3::Double:
 238             m_zeroValues[i] = m_currentBlock->appendIntConstant(m_proc, Origin(), b3Type, 0);
 239             break;
 240         case B3::Void:
 241             m_zeroValues[i] = nullptr;
 242             break;
 243         }
 244     }
 245
 246     if (!!info.memory) {
 247         m_memoryBaseGPR = info.memory.pinnedRegisters().baseMemoryPointer;
 248         m_proc.pinRegister(m_memoryBaseGPR);
 249         ASSERT(!info.memory.pinnedRegisters().sizeRegisters[0].sizeOffset);
 250         m_memorySizeGPR = info.memory.pinnedRegisters().sizeRegisters[0].sizeRegister;
 251         for (const PinnedSizeRegisterInfo& regInfo : info.memory.pinnedRegisters().sizeRegisters)
 252             m_proc.pinRegister(regInfo.sizeRegister);
 253
 254         m_proc.setWasmBoundsCheckGenerator([=] (CCallHelpers& jit, GPRReg pinnedGPR, unsigned) {
 255             AllowMacroScratchRegisterUsage allowScratch(jit);
 256             ASSERT_UNUSED(pinnedGPR, m_memorySizeGPR == pinnedGPR);
 257             jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer();
 258
 259             jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
 260
 261             CCallHelpers::Call call = jit.call();
 262             jit.jumpToExceptionHandler();
 263
 264             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 265                 void (*throwMemoryException)(ExecState*) = [] (ExecState* exec) {
 266                     VM* vm = &exec->vm();
 267                     NativeCallFrameTracer tracer(vm, exec);
 268
 269                     {
 270                         auto throwScope = DECLARE_THROW_SCOPE(*vm);
 271                         JSGlobalObject* globalObject = vm->topJSWebAssemblyInstance->globalObject();
 272
 273                         JSWebAssemblyRuntimeError* error = JSWebAssemblyRuntimeError::create(
 274                             exec, globalObject->WebAssemblyRuntimeErrorStructure(), "Out of bounds memory access");
 275                         throwException(exec, throwScope, error);
 276                     }
 277
 278                     genericUnwind(vm, exec);
 279                     ASSERT(!!vm->callFrameForCatch);
 280                 };
 281
 282                 linkBuffer.link(call, throwMemoryException);
 283             });
 284         });
 285
 286         B3::PatchpointValue* foo = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
 287         foo->setGenerator(
 288             [=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 289                 AllowMacroScratchRegisterUsage allowScratch(jit);
 290             });
 291     }
 292
 293     wasmCallingConvention().setupFrameInPrologue(&compilation->wasmCalleeMoveLocation, m_proc, Origin(), m_currentBlock);
 294
 295     m_instanceValue = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
 296         m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), &m_vm.topJSWebAssemblyInstance));
 297 }
 298
 299 Value* B3IRGenerator::zeroForType(Type type)
 300 {
 301     ASSERT(type != Void);
 302     Value* zeroValue = m_zeroValues[linearizeType(type)];
 303     ASSERT(zeroValue);
 304     return zeroValue;
 305 }
 306
 307 auto B3IRGenerator::addLocal(Type type, uint32_t count) -> PartialResult
 308 {
 309     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(m_locals.size() + count), "can't allocate memory for ", m_locals.size() + count, " locals");
 310
 311     for (uint32_t i = 0; i < count; ++i) {
 312         Variable* local = m_proc.addVariable(toB3Type(type));
 313         m_locals.uncheckedAppend(local);
 314         m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), local, zeroForType(type));
 315     }
 316     return { };
 317 }
 318
 319 auto B3IRGenerator::addArguments(const Vector<Type>& types) -> PartialResult
 320 {
 321     ASSERT(!m_locals.size());
 322     WASM_COMPILE_FAIL_IF(!m_locals.tryReserveCapacity(types.size()), "can't allocate memory for ", types.size(), " arguments");
 323
 324     m_locals.grow(types.size());
 325     wasmCallingConvention().loadArguments(types, m_proc, m_currentBlock, Origin(),
 326         [&] (ExpressionType argument, unsigned i) {
 327             Variable* argumentVariable = m_proc.addVariable(argument->type());
 328             m_locals[i] = argumentVariable;
 329             m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), argumentVariable, argument);
 330         });
 331     return { };
 332 }
 333
 334 auto B3IRGenerator::getLocal(uint32_t index, ExpressionType& result) -> PartialResult
 335 {
 336     ASSERT(m_locals[index]);
 337     result = m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), m_locals[index]);
 338     return { };
 339 }
 340
 341 auto B3IRGenerator::addUnreachable() -> PartialResult
 342 {
 343     B3::PatchpointValue* unreachable = m_currentBlock->appendNew<B3::PatchpointValue>(m_proc, B3::Void, Origin());
 344     unreachable->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 345         jit.breakpoint();
 346     });
 347     unreachable->effects.terminal = true;
 348     return { };
 349 }
 350
 351 auto B3IRGenerator::setLocal(uint32_t index, ExpressionType value) -> PartialResult
 352 {
 353     ASSERT(m_locals[index]);
 354     m_currentBlock->appendNew<VariableValue>(m_proc, B3::Set, Origin(), m_locals[index], value);
 355     return { };
 356 }
 357
 358 auto B3IRGenerator::getGlobal(uint32_t index, ExpressionType& result) -> PartialResult
 359 {
 360     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
 361     result = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, toB3Type(m_info.globals[index].type), Origin(), globalsArray, index * sizeof(Register));
 362     return { };
 363 }
 364
 365 auto B3IRGenerator::setGlobal(uint32_t index, ExpressionType value) -> PartialResult
 366 {
 367     ASSERT(toB3Type(m_info.globals[index].type) == value->type());
 368     Value* globalsArray = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), m_instanceValue, JSWebAssemblyInstance::offsetOfGlobals());
 369     m_currentBlock->appendNew<MemoryValue>(m_proc, Store, Origin(), value, globalsArray, index * sizeof(Register));
 370     return { };
 371 }
 372
 373 inline Value* B3IRGenerator::emitCheckAndPreparePointer(ExpressionType pointer, uint32_t offset, uint32_t sizeOfOperation)
 374 {
 375     ASSERT(m_memoryBaseGPR && m_memorySizeGPR);
 376     ASSERT(sizeOfOperation + offset > offset);
 377     m_currentBlock->appendNew<WasmBoundsCheckValue>(m_proc, Origin(), pointer, m_memorySizeGPR, sizeOfOperation + offset - 1);
 378     pointer = m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), pointer);
 379     return m_currentBlock->appendNew<WasmAddressValue>(m_proc, Origin(), pointer, m_memoryBaseGPR);
 380 }
 381
 382 inline uint32_t sizeOfLoadOp(LoadOpType op)
 383 {
 384     switch (op) {
 385     case LoadOpType::I32Load8S:
 386     case LoadOpType::I32Load8U:
 387     case LoadOpType::I64Load8S:
 388     case LoadOpType::I64Load8U:
 389         return 1;
 390     case LoadOpType::I32Load16S:
 391     case LoadOpType::I64Load16S:
 392     case LoadOpType::I32Load16U:
 393     case LoadOpType::I64Load16U:
 394         return 2;
 395     case LoadOpType::I32Load:
 396     case LoadOpType::I64Load32S:
 397     case LoadOpType::I64Load32U:
 398     case LoadOpType::F32Load:
 399         return 4;
 400     case LoadOpType::I64Load:
 401     case LoadOpType::F64Load:
 402         return 8;
 403     }
 404     RELEASE_ASSERT_NOT_REACHED();
 405 }
 406
 407 inline Value* B3IRGenerator::emitLoadOp(LoadOpType op, Origin origin, ExpressionType pointer, uint32_t offset)
 408 {
 409     switch (op) {
 410     case LoadOpType::I32Load8S: {
 411         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 412     }
 413
 414     case LoadOpType::I64Load8S: {
 415         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8S, origin, pointer, offset);
 416         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 417     }
 418
 419     case LoadOpType::I32Load8U: {
 420         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 421     }
 422
 423     case LoadOpType::I64Load8U: {
 424         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load8Z, origin, pointer, offset);
 425         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 426     }
 427
 428     case LoadOpType::I32Load16S: {
 429         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 430     }
 431     case LoadOpType::I64Load16S: {
 432         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 433         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 434     }
 435
 436     case LoadOpType::I32Load: {
 437         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 438     }
 439
 440     case LoadOpType::I64Load32U: {
 441         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 442         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, origin, value);
 443     }
 444
 445     case LoadOpType::I64Load32S: {
 446         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, origin, pointer);
 447         return m_currentBlock->appendNew<Value>(m_proc, SExt32, origin, value);
 448     }
 449
 450     case LoadOpType::I64Load: {
 451         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int64, origin, pointer);
 452     }
 453
 454     case LoadOpType::F32Load: {
 455         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Float, origin, pointer);
 456     }
 457
 458     case LoadOpType::F64Load: {
 459         return m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Double, origin, pointer);
 460     }
 461
 462     // FIXME: B3 doesn't support Load16Z yet. We should lower to that value when
 463     // it's added. https://bugs.webkit.org/show_bug.cgi?id=165884
 464     case LoadOpType::I32Load16U: {
 465         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 466         return m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
 467                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
 468     }
 469     case LoadOpType::I64Load16U: {
 470         Value* value = m_currentBlock->appendNew<MemoryValue>(m_proc, Load16S, origin, pointer, offset);
 471         Value* partialResult = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(), value,
 472                 m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), 0x0000ffff));
 473
 474         return m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), partialResult);
 475     }
 476     }
 477     RELEASE_ASSERT_NOT_REACHED();
 478 }
 479
 480 auto B3IRGenerator::load(LoadOpType op, ExpressionType pointer, ExpressionType& result, uint32_t offset) -> PartialResult
 481 {
 482     ASSERT(pointer->type() == Int32);
 483
 484     result = emitLoadOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfLoadOp(op)), offset);
 485     return { };
 486 }
 487
 488 inline uint32_t sizeOfStoreOp(StoreOpType op)
 489 {
 490     switch (op) {
 491     case StoreOpType::I32Store8:
 492     case StoreOpType::I64Store8:
 493         return 1;
 494     case StoreOpType::I32Store16:
 495     case StoreOpType::I64Store16:
 496         return 2;
 497     case StoreOpType::I32Store:
 498     case StoreOpType::I64Store32:
 499     case StoreOpType::F32Store:
 500         return 4;
 501     case StoreOpType::I64Store:
 502     case StoreOpType::F64Store:
 503         return 8;
 504     }
 505     RELEASE_ASSERT_NOT_REACHED();
 506 }
 507
 508
 509 inline void B3IRGenerator::emitStoreOp(StoreOpType op, Origin origin, ExpressionType pointer, ExpressionType value, uint32_t offset)
 510 {
 511     switch (op) {
 512     case StoreOpType::I64Store8:
 513         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 514         FALLTHROUGH;
 515
 516     case StoreOpType::I32Store8:
 517         m_currentBlock->appendNew<MemoryValue>(m_proc, Store8, origin, value, pointer, offset);
 518         return;
 519
 520     case StoreOpType::I64Store16:
 521         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 522         FALLTHROUGH;
 523
 524     case StoreOpType::I32Store16:
 525         m_currentBlock->appendNew<MemoryValue>(m_proc, Store16, origin, value, pointer, offset);
 526         return;
 527
 528     case StoreOpType::I64Store32:
 529         value = m_currentBlock->appendNew<Value>(m_proc, Trunc, origin, value);
 530         FALLTHROUGH;
 531
 532     case StoreOpType::I64Store:
 533     case StoreOpType::I32Store:
 534     case StoreOpType::F32Store:
 535     case StoreOpType::F64Store:
 536         m_currentBlock->appendNew<MemoryValue>(m_proc, Store, origin, value, pointer, offset);
 537         return;
 538     }
 539     RELEASE_ASSERT_NOT_REACHED();
 540 }
 541
 542 auto B3IRGenerator::store(StoreOpType op, ExpressionType pointer, ExpressionType value, uint32_t offset) -> PartialResult
 543 {
 544     ASSERT(pointer->type() == Int32);
 545
 546     emitStoreOp(op, Origin(), emitCheckAndPreparePointer(pointer, offset, sizeOfStoreOp(op)), value, offset);
 547     return { };
 548 }
 549
 550 auto B3IRGenerator::addSelect(ExpressionType condition, ExpressionType nonZero, ExpressionType zero, ExpressionType& result) -> PartialResult
 551 {
 552     result = m_currentBlock->appendNew<Value>(m_proc, B3::Select, Origin(), condition, nonZero, zero);
 553     return { };
 554 }
 555
 556 B3IRGenerator::ExpressionType B3IRGenerator::addConstant(Type type, uint64_t value)
 557 {
 558     switch (type) {
 559     case Wasm::I32:
 560         return m_currentBlock->appendNew<Const32Value>(m_proc, Origin(), static_cast<int32_t>(value));
 561     case Wasm::I64:
 562         return m_currentBlock->appendNew<Const64Value>(m_proc, Origin(), value);
 563     case Wasm::F32:
 564         return m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), bitwise_cast<float>(static_cast<int32_t>(value)));
 565     case Wasm::F64:
 566         return m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), bitwise_cast<double>(value));
 567     case Wasm::Void:
 568     case Wasm::Func:
 569     case Wasm::Anyfunc:
 570         break;
 571     }
 572     RELEASE_ASSERT_NOT_REACHED();
 573     return nullptr;
 574 }
 575
 576 B3IRGenerator::ControlData B3IRGenerator::addBlock(Type signature)
 577 {
 578     return ControlData(m_proc, signature, BlockType::Block, m_proc.addBlock());
 579 }
 580
 581 B3IRGenerator::ControlData B3IRGenerator::addLoop(Type signature)
 582 {
 583     BasicBlock* body = m_proc.addBlock();
 584     BasicBlock* continuation = m_proc.addBlock();
 585     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), body);
 586     body->addPredecessor(m_currentBlock);
 587     m_currentBlock = body;
 588     return ControlData(m_proc, signature, BlockType::Loop, continuation, body);
 589 }
 590
 591 auto B3IRGenerator::addIf(ExpressionType condition, Type signature, ControlType& result) -> PartialResult
 592 {
 593     // FIXME: This needs to do some kind of stack passing.
 594
 595     BasicBlock* taken = m_proc.addBlock();
 596     BasicBlock* notTaken = m_proc.addBlock();
 597     BasicBlock* continuation = m_proc.addBlock();
 598
 599     m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 600     m_currentBlock->setSuccessors(FrequentedBlock(taken), FrequentedBlock(notTaken));
 601     taken->addPredecessor(m_currentBlock);
 602     notTaken->addPredecessor(m_currentBlock);
 603
 604     m_currentBlock = taken;
 605     result = ControlData(m_proc, signature, BlockType::If, continuation, notTaken);
 606     return { };
 607 }
 608
 609 auto B3IRGenerator::addElse(ControlData& data, const ExpressionList& currentStack) -> PartialResult
 610 {
 611     unifyValuesWithBlock(currentStack, data.result);
 612     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 613     return addElseToUnreachable(data);
 614 }
 615
 616 auto B3IRGenerator::addElseToUnreachable(ControlData& data) -> PartialResult
 617 {
 618     ASSERT(data.type() == BlockType::If);
 619     m_currentBlock = data.special;
 620     data.convertIfToBlock();
 621     return { };
 622 }
 623
 624 auto B3IRGenerator::addReturn(const ExpressionList& returnValues) -> PartialResult
 625 {
 626     ASSERT(returnValues.size() <= 1);
 627     if (returnValues.size())
 628         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin(), returnValues[0]);
 629     else
 630         m_currentBlock->appendNewControlValue(m_proc, B3::Return, Origin());
 631     return { };
 632 }
 633
 634 auto B3IRGenerator::addBranch(ControlData& data, ExpressionType condition, const ExpressionList& returnValues) -> PartialResult
 635 {
 636     if (data.type() != BlockType::Loop)
 637         unifyValuesWithBlock(returnValues, data.result);
 638
 639     BasicBlock* target = data.targetBlockForBranch();
 640     if (condition) {
 641         BasicBlock* continuation = m_proc.addBlock();
 642         m_currentBlock->appendNew<Value>(m_proc, B3::Branch, Origin(), condition);
 643         m_currentBlock->setSuccessors(FrequentedBlock(target), FrequentedBlock(continuation));
 644         target->addPredecessor(m_currentBlock);
 645         continuation->addPredecessor(m_currentBlock);
 646         m_currentBlock = continuation;
 647     } else {
 648         m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), FrequentedBlock(target));
 649         target->addPredecessor(m_currentBlock);
 650     }
 651
 652     return { };
 653 }
 654
 655 auto B3IRGenerator::addSwitch(ExpressionType condition, const Vector<ControlData*>& targets, ControlData& defaultTarget, const ExpressionList& expressionStack) -> PartialResult
 656 {
 657     for (size_t i = 0; i < targets.size(); ++i)
 658         unifyValuesWithBlock(expressionStack, targets[i]->result);
 659     unifyValuesWithBlock(expressionStack, defaultTarget.result);
 660
 661     SwitchValue* switchValue = m_currentBlock->appendNew<SwitchValue>(m_proc, Origin(), condition);
 662     switchValue->setFallThrough(FrequentedBlock(defaultTarget.targetBlockForBranch()));
 663     for (size_t i = 0; i < targets.size(); ++i)
 664         switchValue->appendCase(SwitchCase(i, FrequentedBlock(targets[i]->targetBlockForBranch())));
 665
 666     return { };
 667 }
 668
 669 auto B3IRGenerator::endBlock(ControlEntry& entry, ExpressionList& expressionStack) -> PartialResult
 670 {
 671     ControlData& data = entry.controlData;
 672
 673     unifyValuesWithBlock(expressionStack, data.result);
 674     m_currentBlock->appendNewControlValue(m_proc, Jump, Origin(), data.continuation);
 675     data.continuation->addPredecessor(m_currentBlock);
 676
 677     return addEndToUnreachable(entry);
 678 }
 679
 680
 681 auto B3IRGenerator::addEndToUnreachable(ControlEntry& entry) -> PartialResult
 682 {
 683     ControlData& data = entry.controlData;
 684     m_currentBlock = data.continuation;
 685
 686     if (data.type() == BlockType::If) {
 687         data.special->appendNewControlValue(m_proc, Jump, Origin(), m_currentBlock);
 688         m_currentBlock->addPredecessor(data.special);
 689     }
 690
 691     for (Variable* result : data.result)
 692         entry.enclosedExpressionStack.append(m_currentBlock->appendNew<VariableValue>(m_proc, B3::Get, Origin(), result));
 693
 694     return { };
 695 }
 696
 697 auto B3IRGenerator::addCall(uint32_t functionIndex, const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
 698 {
 699     ASSERT(signature->arguments.size() == args.size());
 700
 701     Type returnType = signature->returnType;
 702
 703     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
 704         [&] (PatchpointValue* patchpoint) {
 705             patchpoint->effects.writesPinned = true;
 706             patchpoint->effects.readsPinned = true;
 707
 708             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 709                 AllowMacroScratchRegisterUsage allowScratch(jit);
 710
 711                 CCallHelpers::Call call = jit.call();
 712
 713                 jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 714                     m_unlinkedWasmToWasmCalls.append({ linkBuffer.locationOf(call), functionIndex });
 715                 });
 716             });
 717         });
 718     return { };
 719 }
 720
 721 auto B3IRGenerator::addCallIndirect(const Signature* signature, Vector<ExpressionType>& args, ExpressionType& result) -> PartialResult
 722 {
 723     ExpressionType calleeIndex = args.takeLast();
 724     ASSERT(signature->arguments.size() == args.size());
 725
 726     ExpressionType callableFunctionBuffer;
 727     ExpressionType callableFunctionBufferSize;
 728     {
 729         ExpressionType table = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
 730             m_instanceValue, JSWebAssemblyInstance::offsetOfTable());
 731         callableFunctionBuffer = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(),
 732             table, JSWebAssemblyTable::offsetOfFunctions());
 733         callableFunctionBufferSize = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, Int32, Origin(),
 734             table, JSWebAssemblyTable::offsetOfSize());
 735     }
 736
 737     // Check the index we are looking for is valid.
 738     {
 739         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 740             m_currentBlock->appendNew<Value>(m_proc, AboveEqual, Origin(), calleeIndex, callableFunctionBufferSize));
 741
 742         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 743             jit.breakpoint();
 744         });
 745     }
 746
 747     // Compute the offset in the table index space we are looking for.
 748     ExpressionType offset = m_currentBlock->appendNew<Value>(m_proc, Mul, Origin(),
 749         m_currentBlock->appendNew<Value>(m_proc, ZExt32, Origin(), calleeIndex),
 750         m_currentBlock->appendIntConstant(m_proc, Origin(), pointerType(), sizeof(CallableFunction)));
 751     ExpressionType callableFunction = m_currentBlock->appendNew<Value>(m_proc, Add, Origin(), callableFunctionBuffer, offset);
 752
 753     // Check that the CallableFunction is initialized. We trap if it isn't. A null Signature* indicates it's not initialized.
 754     ExpressionType calleeSignature = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, signature));
 755     {
 756         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 757             m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(),
 758                 calleeSignature,
 759                 m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), 0)));
 760
 761         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 762             jit.breakpoint();
 763         });
 764     }
 765
 766     // Check the signature matches the value we expect.
 767     {
 768         ExpressionType expectedSignature = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), signature);
 769         CheckValue* check = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(),
 770             m_currentBlock->appendNew<Value>(m_proc, NotEqual, Origin(), calleeSignature, expectedSignature));
 771
 772         check->setGenerator([] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 773             jit.breakpoint();
 774         });
 775     }
 776
 777     ExpressionType calleeCode = m_currentBlock->appendNew<MemoryValue>(m_proc, Load, pointerType(), Origin(), callableFunction, OBJECT_OFFSETOF(CallableFunction, code));
 778
 779     Type returnType = signature->returnType;
 780     result = wasmCallingConvention().setupCall(m_proc, m_currentBlock, Origin(), args, toB3Type(returnType),
 781         [&] (PatchpointValue* patchpoint) {
 782             patchpoint->effects.writesPinned = true;
 783             patchpoint->effects.readsPinned = true;
 784
 785             patchpoint->append(calleeCode, ValueRep::SomeRegister);
 786
 787             patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) {
 788                 jit.call(params[returnType == Void ? 0 : 1].gpr());
 789             });
 790         });
 791
 792     return { };
 793 }
 794
 795 void B3IRGenerator::unify(Variable* variable, ExpressionType source)
 796 {
 797     m_currentBlock->appendNew<VariableValue>(m_proc, Set, Origin(), variable, source);
 798 }
 799
 800 void B3IRGenerator::unifyValuesWithBlock(const ExpressionList& resultStack, ResultList& result)
 801 {
 802     ASSERT(result.size() <= resultStack.size());
 803
 804     for (size_t i = 0; i < result.size(); ++i)
 805         unify(result[result.size() - 1 - i], resultStack[resultStack.size() - 1 - i]);
 806 }
 807
 808 static void dumpExpressionStack(const CommaPrinter& comma, const B3IRGenerator::ExpressionList& expressionStack)
 809 {
 810     dataLogLn(comma, "ExpressionStack:");
 811     for (const auto& expression : expressionStack)
 812         dataLogLn(comma, *expression);
 813 }
 814
 815 void B3IRGenerator::dump(const Vector<ControlEntry>& controlStack, const ExpressionList& expressionStack)
 816 {
 817     dataLogLn("Processing Graph:");
 818     dataLog(m_proc);
 819     dataLogLn("With current block:", *m_currentBlock);
 820     dataLogLn("Control stack:");
 821     for (auto& data : controlStack) {
 822         dataLogLn("  ", data.controlData);
 823         if (data.enclosedExpressionStack.size()) {
 824             CommaPrinter comma("    ", "  with ");
 825             dumpExpressionStack(comma, data.enclosedExpressionStack);
 826         }
 827     }
 828
 829     CommaPrinter comma("  ", "");
 830     dumpExpressionStack(comma, expressionStack);
 831     dataLogLn("\n");
 832 }
 833
 834 static std::unique_ptr<Compilation> createJSToWasmWrapper(VM& vm, WasmInternalFunction& function, const Signature* signature, MacroAssemblerCodePtr mainFunction, const MemoryInformation& memory)
 835 {
 836     Procedure proc;
 837     BasicBlock* block = proc.addBlock();
 838
 839     Origin origin;
 840
 841     jscCallingConvention().setupFrameInPrologue(&function.jsToWasmCalleeMoveLocation, proc, origin, block);
 842
 843     Value* framePointer = block->appendNew<B3::Value>(proc, B3::FramePointer, origin);
 844     Value* offSetOfArgumentCount = block->appendNew<Const64Value>(proc, origin, CallFrameSlot::argumentCount * sizeof(Register));
 845     Value* argumentCount = block->appendNew<MemoryValue>(proc, Load, Int32, origin,
 846         block->appendNew<Value>(proc, Add, origin, framePointer, offSetOfArgumentCount));
 847
 848     Value* expectedArgumentCount = block->appendNew<Const32Value>(proc, origin, signature->arguments.size());
 849
 850     CheckValue* argumentCountCheck = block->appendNew<CheckValue>(proc, Check, origin,
 851         block->appendNew<Value>(proc, Above, origin, expectedArgumentCount, argumentCount));
 852
 853     argumentCountCheck->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
 854         jit.breakpoint();
 855     });
 856
 857     // Move memory values to the approriate places, if needed.
 858     Value* baseMemory = nullptr;
 859     Vector<Value*> sizes;
 860     if (!!memory) {
 861         baseMemory = block->appendNew<MemoryValue>(proc, Load, Int64, Origin(),
 862             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemoryPointer));
 863         Value* size = block->appendNew<MemoryValue>(proc, Load, Int32, Origin(),
 864             block->appendNew<ConstPtrValue>(proc, Origin(), &vm.topWasmMemorySize));
 865         sizes.reserveCapacity(memory.pinnedRegisters().sizeRegisters.size());
 866         for (auto info : memory.pinnedRegisters().sizeRegisters) {
 867             sizes.append(block->appendNew<Value>(proc, Sub, origin, size,
 868                 block->appendNew<Const32Value>(proc, origin, info.sizeOffset)));
 869         }
 870     }
 871
 872     // Get our arguments.
 873     Vector<Value*> arguments;
 874     jscCallingConvention().loadArguments(signature->arguments, proc, block, origin, [&] (Value* argument, unsigned) {
 875         arguments.append(argument);
 876     });
 877
 878     // Move the arguments into place.
 879     Value* result = wasmCallingConvention().setupCall(proc, block, origin, arguments, toB3Type(signature->returnType), [&] (PatchpointValue* patchpoint) {
 880         if (!!memory) {
 881             ASSERT(sizes.size() == memory.pinnedRegisters().sizeRegisters.size());
 882             patchpoint->append(ConstrainedValue(baseMemory, ValueRep::reg(memory.pinnedRegisters().baseMemoryPointer)));
 883             for (unsigned i = 0; i < sizes.size(); ++i)
 884                 patchpoint->append(ConstrainedValue(sizes[i], ValueRep::reg(memory.pinnedRegisters().sizeRegisters[i].sizeRegister)));
 885         }
 886
 887         patchpoint->setGenerator([=] (CCallHelpers& jit, const B3::StackmapGenerationParams&) {
 888             AllowMacroScratchRegisterUsage allowScratch(jit);
 889
 890             CCallHelpers::Call call = jit.call();
 891             jit.addLinkTask([=] (LinkBuffer& linkBuffer) {
 892                 linkBuffer.link(call, FunctionPtr(mainFunction.executableAddress()));
 893             });
 894         });
 895     });
 896
 897     // Return the result, if needed.
 898     switch (signature->returnType) {
 899     case Wasm::Void:
 900         block->appendNewControlValue(proc, B3::Return, origin);
 901         break;
 902     case Wasm::F32:
 903     case Wasm::F64:
 904         result = block->appendNew<Value>(proc, BitwiseCast, origin, result);
 905         FALLTHROUGH;
 906     case Wasm::I32:
 907     case Wasm::I64:
 908         block->appendNewControlValue(proc, B3::Return, origin, result);
 909         break;
 910     case Wasm::Func:
 911     case Wasm::Anyfunc:
 912         RELEASE_ASSERT_NOT_REACHED();
 913     }
 914
 915     auto jsEntrypoint = std::make_unique<Compilation>(vm, proc);
 916     function.jsToWasmEntrypoint.calleeSaveRegisters = proc.calleeSaveRegisters();
 917     return jsEntrypoint;
 918 }
 919
 920 Expected<std::unique_ptr<WasmInternalFunction>, String> parseAndCompile(VM& vm, const uint8_t* functionStart, size_t functionLength, const Signature* signature, Vector<UnlinkedWasmToWasmCall>& unlinkedWasmToWasmCalls, const ImmutableFunctionIndexSpace& functionIndexSpace, const ModuleInformation& info, unsigned optLevel)
 921 {
 922     auto result = std::make_unique<WasmInternalFunction>();
 923
 924     Procedure procedure;
 925     B3IRGenerator context(vm, info, procedure, result.get(), unlinkedWasmToWasmCalls, functionIndexSpace);
 926     FunctionParser<B3IRGenerator> parser(context, functionStart, functionLength, signature, functionIndexSpace, info);
 927     WASM_FAIL_IF_HELPER_FAILS(parser.parse());
 928
 929     procedure.resetReachability();
 930     validate(procedure, "After parsing:\n");
 931
 932     if (verbose)
 933         dataLog("Pre SSA: ", procedure);
 934     fixSSA(procedure);
 935     if (verbose)
 936         dataLog("Post SSA: ", procedure);
 937
 938     result->wasmEntrypoint.compilation = std::make_unique<Compilation>(vm, procedure, optLevel);
 939     result->wasmEntrypoint.calleeSaveRegisters = procedure.calleeSaveRegisters();
 940     result->jsToWasmEntrypoint.compilation = createJSToWasmWrapper(vm, *result, signature, result->wasmEntrypoint.compilation->code(), info.memory);
 941     return WTFMove(result);
 942 }
 943
 944 // Custom wasm ops. These are the ones too messy to do in wasm.json.
 945
 946 template<>
 947 auto B3IRGenerator::addOp<OpType::I32Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
 948 {
 949     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
 950     patchpoint->append(arg, ValueRep::SomeRegister);
 951     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 952         jit.countTrailingZeros32(params[1].gpr(), params[0].gpr());
 953     });
 954     patchpoint->effects = Effects::none();
 955     result = patchpoint;
 956     return { };
 957 }
 958
 959 template<>
 960 auto B3IRGenerator::addOp<OpType::I64Ctz>(ExpressionType arg, ExpressionType& result) -> PartialResult
 961 {
 962     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
 963     patchpoint->append(arg, ValueRep::SomeRegister);
 964     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
 965         jit.countTrailingZeros64(params[1].gpr(), params[0].gpr());
 966     });
 967     patchpoint->effects = Effects::none();
 968     result = patchpoint;
 969     return { };
 970 }
 971
 972 template<>
 973 auto B3IRGenerator::addOp<OpType::I32Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
 974 {
 975     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 976     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 977     uint32_t (*popcount)(int32_t) = [] (int32_t value) -> uint32_t { return __builtin_popcount(value); };
 978     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 979     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int32, Origin(), Effects::none(), funcAddress, arg);
 980     return { };
 981 }
 982
 983 template<>
 984 auto B3IRGenerator::addOp<OpType::I64Popcnt>(ExpressionType arg, ExpressionType& result) -> PartialResult
 985 {
 986     // FIXME: This should use the popcnt instruction if SSE4 is available but we don't have code to detect SSE4 yet.
 987     // see: https://bugs.webkit.org/show_bug.cgi?id=165363
 988     uint64_t (*popcount)(int64_t) = [] (int64_t value) -> uint64_t { return __builtin_popcountll(value); };
 989     Value* funcAddress = m_currentBlock->appendNew<ConstPtrValue>(m_proc, Origin(), bitwise_cast<void*>(popcount));
 990     result = m_currentBlock->appendNew<CCallValue>(m_proc, Int64, Origin(), Effects::none(), funcAddress, arg);
 991     return { };
 992 }
 993
 994 template<>
 995 auto B3IRGenerator::addOp<F64ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
 996 {
 997     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
 998     if (isX86())
 999         patchpoint->numGPScratchRegisters = 1;
1000     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
1001     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1002         AllowMacroScratchRegisterUsage allowScratch(jit);
1003 #if CPU(X86_64)
1004         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1005 #else
1006         jit.convertUInt64ToDouble(params[1].gpr(), params[0].fpr());
1007 #endif
1008     });
1009     patchpoint->effects = Effects::none();
1010     result = patchpoint;
1011     return { };
1012 }
1013
1014 template<>
1015 auto B3IRGenerator::addOp<OpType::F32ConvertUI64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1016 {
1017     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1018     if (isX86())
1019         patchpoint->numGPScratchRegisters = 1;
1020     patchpoint->append(ConstrainedValue(arg, ValueRep::WarmAny));
1021     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1022         AllowMacroScratchRegisterUsage allowScratch(jit);
1023 #if CPU(X86_64)
1024         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr(), params.gpScratch(0));
1025 #else
1026         jit.convertUInt64ToFloat(params[1].gpr(), params[0].fpr());
1027 #endif
1028     });
1029     patchpoint->effects = Effects::none();
1030     result = patchpoint;
1031     return { };
1032 }
1033
1034 template<>
1035 auto B3IRGenerator::addOp<OpType::F64Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1036 {
1037     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1038     patchpoint->append(arg, ValueRep::SomeRegister);
1039     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1040         jit.roundTowardNearestIntDouble(params[1].fpr(), params[0].fpr());
1041     });
1042     patchpoint->effects = Effects::none();
1043     result = patchpoint;
1044     return { };
1045 }
1046
1047 template<>
1048 auto B3IRGenerator::addOp<OpType::F32Nearest>(ExpressionType arg, ExpressionType& result) -> PartialResult
1049 {
1050     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1051     patchpoint->append(arg, ValueRep::SomeRegister);
1052     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1053         jit.roundTowardNearestIntFloat(params[1].fpr(), params[0].fpr());
1054     });
1055     patchpoint->effects = Effects::none();
1056     result = patchpoint;
1057     return { };
1058 }
1059
1060 template<>
1061 auto B3IRGenerator::addOp<OpType::F64Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1062 {
1063     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Double, Origin());
1064     patchpoint->append(arg, ValueRep::SomeRegister);
1065     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1066         jit.roundTowardZeroDouble(params[1].fpr(), params[0].fpr());
1067     });
1068     patchpoint->effects = Effects::none();
1069     result = patchpoint;
1070     return { };
1071 }
1072
1073 template<>
1074 auto B3IRGenerator::addOp<OpType::F32Trunc>(ExpressionType arg, ExpressionType& result) -> PartialResult
1075 {
1076     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Float, Origin());
1077     patchpoint->append(arg, ValueRep::SomeRegister);
1078     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1079         jit.roundTowardZeroFloat(params[1].fpr(), params[0].fpr());
1080     });
1081     patchpoint->effects = Effects::none();
1082     result = patchpoint;
1083     return { };
1084 }
1085
1086 template<>
1087 auto B3IRGenerator::addOp<OpType::I32TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1088 {
1089     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int32_t>::min()));
1090     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()));
1091     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1092         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1093         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1094     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1095     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1096     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1097         jit.breakpoint();
1098     });
1099     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1100     patchpoint->append(arg, ValueRep::SomeRegister);
1101     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1102         jit.truncateDoubleToInt32(params[1].fpr(), params[0].gpr());
1103     });
1104     patchpoint->effects = Effects::none();
1105     result = patchpoint;
1106     return { };
1107 }
1108
1109 template<>
1110 auto B3IRGenerator::addOp<OpType::I32TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1111 {
1112     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int32_t>::min()));
1113     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()));
1114     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1115         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1116         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1117     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1118     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1119     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1120         jit.breakpoint();
1121     });
1122     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1123     patchpoint->append(arg, ValueRep::SomeRegister);
1124     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1125         jit.truncateFloatToInt32(params[1].fpr(), params[0].gpr());
1126     });
1127     patchpoint->effects = Effects::none();
1128     result = patchpoint;
1129     return { };
1130 }
1131
1132
1133 template<>
1134 auto B3IRGenerator::addOp<OpType::I32TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1135 {
1136     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int32_t>::min()) * -2.0);
1137     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1138     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1139         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1140         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1141     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1142     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1143     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1144         jit.breakpoint();
1145     });
1146     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1147     patchpoint->append(arg, ValueRep::SomeRegister);
1148     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1149         jit.truncateDoubleToUint32(params[1].fpr(), params[0].gpr());
1150     });
1151     patchpoint->effects = Effects::none();
1152     result = patchpoint;
1153     return { };
1154 }
1155
1156 template<>
1157 auto B3IRGenerator::addOp<OpType::I32TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1158 {
1159     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int32_t>::min()) * -2.0);
1160     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1161     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1162         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1163         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1164     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1165     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1166     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1167         jit.breakpoint();
1168     });
1169     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int32, Origin());
1170     patchpoint->append(arg, ValueRep::SomeRegister);
1171     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1172         jit.truncateFloatToUint32(params[1].fpr(), params[0].gpr());
1173     });
1174     patchpoint->effects = Effects::none();
1175     result = patchpoint;
1176     return { };
1177 }
1178
1179 template<>
1180 auto B3IRGenerator::addOp<OpType::I64TruncSF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1181 {
1182     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -static_cast<double>(std::numeric_limits<int64_t>::min()));
1183     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()));
1184     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1185         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1186         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1187     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1188     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1189     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1190         jit.breakpoint();
1191     });
1192     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1193     patchpoint->append(arg, ValueRep::SomeRegister);
1194     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1195         jit.truncateDoubleToInt64(params[1].fpr(), params[0].gpr());
1196     });
1197     patchpoint->effects = Effects::none();
1198     result = patchpoint;
1199     return { };
1200 }
1201
1202 template<>
1203 auto B3IRGenerator::addOp<OpType::I64TruncUF64>(ExpressionType arg, ExpressionType& result) -> PartialResult
1204 {
1205     Value* max = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<int64_t>::min()) * -2.0);
1206     Value* min = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), -1.0);
1207     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1208         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1209         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1210     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1211     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1212     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1213         jit.breakpoint();
1214     });
1215
1216     Value* constant;
1217     if (isX86()) {
1218         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1219         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1220         // so we can pool them if needed.
1221         constant = m_currentBlock->appendNew<ConstDoubleValue>(m_proc, Origin(), static_cast<double>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1222     }
1223     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1224     patchpoint->append(arg, ValueRep::SomeRegister);
1225     if (isX86()) {
1226         patchpoint->append(constant, ValueRep::SomeRegister);
1227         patchpoint->numFPScratchRegisters = 1;
1228     }
1229     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1230         AllowMacroScratchRegisterUsage allowScratch(jit);
1231         FPRReg scratch = InvalidFPRReg;
1232         FPRReg constant = InvalidFPRReg;
1233         if (isX86()) {
1234             scratch = params.fpScratch(0);
1235             constant = params[2].fpr();
1236         }
1237         jit.truncateDoubleToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1238     });
1239     patchpoint->effects = Effects::none();
1240     result = patchpoint;
1241     return { };
1242 }
1243
1244 template<>
1245 auto B3IRGenerator::addOp<OpType::I64TruncSF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1246 {
1247     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -static_cast<float>(std::numeric_limits<int64_t>::min()));
1248     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()));
1249     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1250         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1251         m_currentBlock->appendNew<Value>(m_proc, GreaterEqual, Origin(), arg, min));
1252     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1253     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1254     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1255         jit.breakpoint();
1256     });
1257     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1258     patchpoint->append(arg, ValueRep::SomeRegister);
1259     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1260         jit.truncateFloatToInt64(params[1].fpr(), params[0].gpr());
1261     });
1262     patchpoint->effects = Effects::none();
1263     result = patchpoint;
1264     return { };
1265 }
1266
1267 template<>
1268 auto B3IRGenerator::addOp<OpType::I64TruncUF32>(ExpressionType arg, ExpressionType& result) -> PartialResult
1269 {
1270     Value* max = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<int64_t>::min()) * -2.0);
1271     Value* min = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), -1.0);
1272     Value* outOfBounds = m_currentBlock->appendNew<Value>(m_proc, BitAnd, Origin(),
1273         m_currentBlock->appendNew<Value>(m_proc, LessThan, Origin(), arg, max),
1274         m_currentBlock->appendNew<Value>(m_proc, GreaterThan, Origin(), arg, min));
1275     outOfBounds = m_currentBlock->appendNew<Value>(m_proc, Equal, Origin(), outOfBounds, zeroForType(I32));
1276     CheckValue* trap = m_currentBlock->appendNew<CheckValue>(m_proc, Check, Origin(), outOfBounds);
1277     trap->setGenerator([] (CCallHelpers& jit, const StackmapGenerationParams&) {
1278         jit.breakpoint();
1279     });
1280
1281     Value* constant;
1282     if (isX86()) {
1283         // Since x86 doesn't have an instruction to convert floating points to unsigned integers, we at least try to do the smart thing if
1284         // the numbers are would be positive anyway as a signed integer. Since we cannot materialize constants into fprs we have b3 do it
1285         // so we can pool them if needed.
1286         constant = m_currentBlock->appendNew<ConstFloatValue>(m_proc, Origin(), static_cast<float>(std::numeric_limits<uint64_t>::max() - std::numeric_limits<int64_t>::max()));
1287     }
1288     PatchpointValue* patchpoint = m_currentBlock->appendNew<PatchpointValue>(m_proc, Int64, Origin());
1289     patchpoint->append(arg, ValueRep::SomeRegister);
1290     if (isX86()) {
1291         patchpoint->append(constant, ValueRep::SomeRegister);
1292         patchpoint->numFPScratchRegisters = 1;
1293     }
1294     patchpoint->setGenerator([=] (CCallHelpers& jit, const StackmapGenerationParams& params) {
1295         AllowMacroScratchRegisterUsage allowScratch(jit);
1296         FPRReg scratch = InvalidFPRReg;
1297         FPRReg constant = InvalidFPRReg;
1298         if (isX86()) {
1299             scratch = params.fpScratch(0);
1300             constant = params[2].fpr();
1301         }
1302         jit.truncateFloatToUint64(params[1].fpr(), params[0].gpr(), scratch, constant);
1303     });
1304     patchpoint->effects = Effects::none();
1305     result = patchpoint;
1306     return { };
1307 }
1308
1309 } } // namespace JSC::Wasm
1310
1311 #include "WasmB3IRGeneratorInlines.h"
1312
1313 #endif // ENABLE(WEBASSEMBLY)