Try ripping out inferred types because it might be a performance improvement
[WebKit-https.git] / Source / JavaScriptCore / dfg / DFGObjectAllocationSinkingPhase.cpp
index 36bb13f..b99a4dd 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2015 Apple Inc. All rights reserved.
+ * Copyright (C) 2015-2018 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 #include "DFGSSACalculator.h"
 #include "DFGValidate.h"
 #include "JSCInlines.h"
-
-#include <list>
+#include <wtf/StdList.h>
 
 namespace JSC { namespace DFG {
 
 namespace {
 
-bool verbose = false;
+namespace DFGObjectAllocationSinkingPhaseInternal {
+static const bool verbose = false;
+}
 
 // In order to sink object cycles, we use a points-to analysis coupled
 // with an escape analysis. This analysis is actually similar to an
@@ -139,7 +140,9 @@ public:
     // once it is escaped if it still has pointers to it in order to
     // replace any use of those pointers by the corresponding
     // materialization
-    enum class Kind { Escaped, Object, Activation, Function, ArrowFunction, GeneratorFunction };
+    enum class Kind { Escaped, Object, Activation, Function, GeneratorFunction, AsyncFunction, AsyncGeneratorFunction, RegExpObject };
+
+    using Fields = HashMap<PromotedLocationDescriptor, Node*>;
 
     explicit Allocation(Node* identifier = nullptr, Kind kind = Kind::Escaped)
         : m_identifier(identifier)
@@ -148,7 +151,12 @@ public:
     }
 
 
-    const HashMap<PromotedLocationDescriptor, Node*>& fields() const
+    const Fields& fields() const
+    {
+        return m_fields;
+    }
+
+    Fields& fields()
     {
         return m_fields;
     }
@@ -186,28 +194,29 @@ public:
         }
     }
 
-    Allocation& setStructures(const StructureSet& structures)
+    Allocation& setStructures(const RegisteredStructureSet& structures)
     {
         ASSERT(hasStructures() && !structures.isEmpty());
         m_structures = structures;
         return *this;
     }
 
-    Allocation& mergeStructures(const StructureSet& structures)
+    Allocation& mergeStructures(const RegisteredStructureSet& structures)
     {
         ASSERT(hasStructures() || structures.isEmpty());
         m_structures.merge(structures);
         return *this;
     }
 
-    Allocation& filterStructures(const StructureSet& structures)
+    Allocation& filterStructures(const RegisteredStructureSet& structures)
     {
         ASSERT(hasStructures());
         m_structures.filter(structures);
+        RELEASE_ASSERT(!m_structures.isEmpty());
         return *this;
     }
 
-    const StructureSet& structures() const
+    const RegisteredStructureSet& structures() const
     {
         return m_structures;
     }
@@ -233,7 +242,12 @@ public:
 
     bool isFunctionAllocation() const
     {
-        return m_kind == Kind::Function || m_kind == Kind::ArrowFunction || m_kind == Kind::GeneratorFunction;
+        return m_kind == Kind::Function || m_kind == Kind::GeneratorFunction || m_kind == Kind::AsyncFunction;
+    }
+
+    bool isRegExpObjectAllocation() const
+    {
+        return m_kind == Kind::RegExpObject;
     }
 
     bool operator==(const Allocation& other) const
@@ -268,22 +282,30 @@ public:
         case Kind::Function:
             out.print("Function");
             break;
-                
-        case Kind::ArrowFunction:
-            out.print("ArrowFunction");
-            break;
 
         case Kind::GeneratorFunction:
             out.print("GeneratorFunction");
             break;
 
+        case Kind::AsyncFunction:
+            out.print("AsyncFunction");
+            break;
+
+        case Kind::AsyncGeneratorFunction:
+            out.print("AsyncGeneratorFunction");
+            break;
+
         case Kind::Activation:
             out.print("Activation");
             break;
+
+        case Kind::RegExpObject:
+            out.print("RegExpObject");
+            break;
         }
         out.print("Allocation(");
         if (!m_structures.isEmpty())
-            out.print(inContext(m_structures, context));
+            out.print(inContext(m_structures.toStructureSet(), context));
         if (!m_fields.isEmpty()) {
             if (!m_structures.isEmpty())
                 out.print(", ");
@@ -295,8 +317,8 @@ public:
 private:
     Node* m_identifier; // This is the actual node that created the allocation
     Kind m_kind;
-    HashMap<PromotedLocationDescriptor, Node*> m_fields;
-    StructureSet m_structures;
+    Fields m_fields;
+    RegisteredStructureSet m_structures;
 };
 
 class LocalHeap {
@@ -441,7 +463,7 @@ public:
             return;
         }
 
-        HashSet<Node*> toEscape;
+        NodeSet toEscape;
 
         for (auto& allocationEntry : other.m_allocations)
             m_allocations.add(allocationEntry.key, allocationEntry.value);
@@ -458,29 +480,16 @@ public:
                 continue;
 
             if (allocationEntry.value.kind() != allocationIter->value.kind()) {
-                toEscape.add(allocationEntry.key);
+                toEscape.addVoid(allocationEntry.key);
                 for (const auto& fieldEntry : allocationIter->value.fields())
-                    toEscape.add(fieldEntry.value);
+                    toEscape.addVoid(fieldEntry.value);
             } else {
-                mergePointerSets(
-                    allocationEntry.value.fields(), allocationIter->value.fields(),
-                    [&] (Node* identifier) {
-                        toEscape.add(identifier);
-                    },
-                    [&] (PromotedLocationDescriptor field) {
-                        allocationEntry.value.remove(field);
-                    });
+                mergePointerSets(allocationEntry.value.fields(), allocationIter->value.fields(), toEscape);
                 allocationEntry.value.mergeStructures(allocationIter->value.structures());
             }
         }
 
-        mergePointerSets(m_pointers, other.m_pointers,
-            [&] (Node* identifier) {
-                toEscape.add(identifier);
-            },
-            [&] (Node* field) {
-                m_pointers.remove(field);
-            });
+        mergePointerSets(m_pointers, other.m_pointers, toEscape);
 
         for (Node* identifier : toEscape)
             escapeAllocation(identifier);
@@ -499,16 +508,12 @@ public:
         assertIsValid();
     }
 
-    void pruneByLiveness(const HashSet<Node*>& live)
+    void pruneByLiveness(const NodeSet& live)
     {
-        Vector<Node*> toRemove;
-        for (const auto& entry : m_pointers) {
-            if (!live.contains(entry.key))
-                toRemove.append(entry.key);
-        }
-        for (Node* node : toRemove)
-            m_pointers.remove(node);
-
+        m_pointers.removeIf(
+            [&] (const auto& entry) {
+                return !live.contains(entry.key);
+            });
         prune();
     }
 
@@ -611,30 +616,30 @@ private:
     //  3: GetByOffset(@0, x)
     //  4: GetByOffset(@3, val)
     //  -: Return(@4)
-    template<typename Key, typename EscapeFunctor, typename RemoveFunctor>
-    void mergePointerSets(
-        const HashMap<Key, Node*>& my, const HashMap<Key, Node*>& their,
-        const EscapeFunctor& escape, const RemoveFunctor& remove)
+    template<typename Key>
+    static void mergePointerSets(HashMap<Key, Node*>& my, const HashMap<Key, Node*>& their, NodeSet& toEscape)
     {
-        Vector<Key> toRemove;
-        for (const auto& entry : my) {
+        auto escape = [&] (Node* identifier) {
+            toEscape.addVoid(identifier);
+        };
+
+        for (const auto& entry : their) {
+            if (!my.contains(entry.key))
+                escape(entry.value);
+        }
+        my.removeIf([&] (const auto& entry) {
             auto iter = their.find(entry.key);
             if (iter == their.end()) {
-                toRemove.append(entry.key);
                 escape(entry.value);
-            } else if (iter->value != entry.value) {
-                toRemove.append(entry.key);
+                return true;
+            }
+            if (iter->value != entry.value) {
                 escape(entry.value);
                 escape(iter->value);
+                return true;
             }
-        }
-        for (const auto& entry : their) {
-            if (my.contains(entry.key))
-                continue;
-            escape(entry.value);
-        }
-        for (Key key : toRemove)
-            remove(key);
+            return false;
+        });
     }
 
     void escapeAllocation(Node* identifier)
@@ -655,9 +660,9 @@ private:
 
     void prune()
     {
-        HashSet<Node*> reachable;
+        NodeSet reachable;
         for (const auto& entry : m_pointers)
-            reachable.add(entry.value);
+            reachable.addVoid(entry.value);
 
         // Repeatedly mark as reachable allocations in fields of other
         // reachable allocations
@@ -676,15 +681,10 @@ private:
         }
 
         // Remove unreachable allocations
-        {
-            Vector<Node*> toRemove;
-            for (const auto& entry : m_allocations) {
-                if (!reachable.contains(entry.key))
-                    toRemove.append(entry.key);
-            }
-            for (Node* identifier : toRemove)
-                m_allocations.remove(identifier);
-        }
+        m_allocations.removeIf(
+            [&] (const auto& entry) {
+                return !reachable.contains(entry.key);
+            });
     }
 
     bool m_reached = false;
@@ -713,7 +713,7 @@ public:
         if (!performSinking())
             return false;
 
-        if (verbose) {
+        if (DFGObjectAllocationSinkingPhaseInternal::verbose) {
             dataLog("Graph after elimination:\n");
             m_graph.dump();
         }
@@ -726,7 +726,7 @@ private:
     {
         m_graph.computeRefCounts();
         m_graph.initializeNodeOwners();
-        m_graph.ensureDominators();
+        m_graph.ensureSSADominators();
         performLivenessAnalysis(m_graph);
         performOSRAvailabilityAnalysis(m_graph);
         m_combinedLiveness = CombinedLiveness(m_graph);
@@ -738,7 +738,7 @@ private:
             graphBeforeSinking = out.toCString();
         }
 
-        if (verbose) {
+        if (DFGObjectAllocationSinkingPhaseInternal::verbose) {
             dataLog("Graph before elimination:\n");
             m_graph.dump();
         }
@@ -748,7 +748,7 @@ private:
         if (!determineSinkCandidates())
             return false;
 
-        if (verbose) {
+        if (DFGObjectAllocationSinkingPhaseInternal::verbose) {
             for (BasicBlock* block : m_graph.blocksInNaturalOrder()) {
                 dataLog("Heap at head of ", *block, ": \n", m_heapAtHead[block]);
                 dataLog("Heap at tail of ", *block, ": \n", m_heapAtTail[block]);
@@ -756,6 +756,7 @@ private:
         }
 
         promoteLocalHeap();
+        removeICStatusFilters();
 
         if (Options::validateGraphAtEachPhase())
             DFG::validate(m_graph, DumpGraph, graphBeforeSinking);
@@ -769,7 +770,7 @@ private:
 
         bool changed;
         do {
-            if (verbose)
+            if (DFGObjectAllocationSinkingPhaseInternal::verbose)
                 dataLog("Doing iteration of escape analysis.\n");
             changed = false;
 
@@ -832,28 +833,40 @@ private:
             target = &m_heap.newAllocation(node, Allocation::Kind::Object);
             target->setStructures(node->structure());
             writes.add(
-                StructurePLoc, LazyNode(m_graph.freeze(node->structure())));
+                StructurePLoc, LazyNode(m_graph.freeze(node->structure().get())));
             break;
 
         case NewFunction:
-        case NewArrowFunction:
-        case NewGeneratorFunction: {
+        case NewGeneratorFunction:
+        case NewAsyncGeneratorFunction:
+        case NewAsyncFunction: {
             if (isStillValid(node->castOperand<FunctionExecutable*>()->singletonFunction())) {
                 m_heap.escape(node->child1().node());
                 break;
             }
-            
+
             if (node->op() == NewGeneratorFunction)
                 target = &m_heap.newAllocation(node, Allocation::Kind::GeneratorFunction);
-            else if (node->op() == NewArrowFunction)
-                target = &m_heap.newAllocation(node, Allocation::Kind::ArrowFunction);
+            else if (node->op() == NewAsyncFunction)
+                target = &m_heap.newAllocation(node, Allocation::Kind::AsyncFunction);
+            else if (node->op() == NewAsyncGeneratorFunction)
+                target = &m_heap.newAllocation(node, Allocation::Kind::AsyncGeneratorFunction);
             else
                 target = &m_heap.newAllocation(node, Allocation::Kind::Function);
+
             writes.add(FunctionExecutablePLoc, LazyNode(node->cellOperand()));
             writes.add(FunctionActivationPLoc, LazyNode(node->child1().node()));
             break;
         }
 
+        case NewRegexp: {
+            target = &m_heap.newAllocation(node, Allocation::Kind::RegExpObject);
+
+            writes.add(RegExpObjectRegExpPLoc, LazyNode(node->cellOperand()));
+            writes.add(RegExpObjectLastIndexPLoc, LazyNode(node->child1().node()));
+            break;
+        }
+
         case CreateActivation: {
             if (isStillValid(node->castOperand<SymbolTable*>()->singletonScope())) {
                 m_heap.escape(node->child1().node());
@@ -864,11 +877,10 @@ private:
             writes.add(ActivationScopePLoc, LazyNode(node->child1().node()));
             {
                 SymbolTable* symbolTable = node->castOperand<SymbolTable*>();
-                ConcurrentJITLocker locker(symbolTable->m_lock);
                 LazyNode initialValue(m_graph.freeze(node->initializationValueForActivation()));
-                for (auto iter = symbolTable->begin(locker), end = symbolTable->end(locker); iter != end; ++iter) {
+                for (unsigned offset = 0; offset < symbolTable->scopeSize(); ++offset) {
                     writes.add(
-                        PromotedLocationDescriptor(ClosureVarPLoc, iter->value.scopeOffset().offset()),
+                        PromotedLocationDescriptor(ClosureVarPLoc, offset),
                         initialValue);
                 }
             }
@@ -878,16 +890,25 @@ private:
         case PutStructure:
             target = m_heap.onlyLocalAllocation(node->child1().node());
             if (target && target->isObjectAllocation()) {
-                writes.add(StructurePLoc, LazyNode(m_graph.freeze(JSValue(node->transition()->next))));
+                writes.add(StructurePLoc, LazyNode(m_graph.freeze(JSValue(node->transition()->next.get()))));
                 target->setStructures(node->transition()->next);
             } else
                 m_heap.escape(node->child1().node());
             break;
 
+        case CheckStructureOrEmpty:
         case CheckStructure: {
             Allocation* allocation = m_heap.onlyLocalAllocation(node->child1().node());
             if (allocation && allocation->isObjectAllocation()) {
-                allocation->filterStructures(node->structureSet());
+                RegisteredStructureSet filteredStructures = allocation->structures();
+                filteredStructures.filter(node->structureSet());
+                if (filteredStructures.isEmpty()) {
+                    // FIXME: Write a test for this:
+                    // https://bugs.webkit.org/show_bug.cgi?id=174322
+                    m_heap.escape(node->child1().node());
+                    break;
+                }
+                allocation->setStructures(filteredStructures);
                 if (Node* value = heapResolve(PromotedHeapLocation(allocation->identifier(), StructurePLoc)))
                     node->convertToCheckStructureImmediate(value);
             } else
@@ -911,7 +932,7 @@ private:
             Allocation* allocation = m_heap.onlyLocalAllocation(node->child1().node());
             if (allocation && allocation->isObjectAllocation()) {
                 MultiGetByOffsetData& data = node->multiGetByOffsetData();
-                StructureSet validStructures;
+                RegisteredStructureSet validStructures;
                 bool hasInvalidStructures = false;
                 for (const auto& multiGetByOffsetCase : data.cases) {
                     if (!allocation->structures().overlaps(multiGetByOffsetCase.set()))
@@ -931,7 +952,7 @@ private:
                         RELEASE_ASSERT_NOT_REACHED();
                     }
                 }
-                if (hasInvalidStructures) {
+                if (hasInvalidStructures || validStructures.isEmpty()) {
                     m_heap.escape(node->child1().node());
                     break;
                 }
@@ -939,7 +960,7 @@ private:
                 PromotedHeapLocation location(NamedPropertyPLoc, allocation->identifier(), identifierNumber);
                 if (Node* value = heapResolve(location)) {
                     if (allocation->structures().isSubsetOf(validStructures))
-                        node->replaceWith(value);
+                        node->replaceWithWithoutChecks(value);
                     else {
                         Node* structure = heapResolve(PromotedHeapLocation(allocation->identifier(), StructurePLoc));
                         ASSERT(structure);
@@ -1028,7 +1049,28 @@ private:
                 m_heap.escape(node->child1().node());
             break;
 
+        case GetRegExpObjectLastIndex:
+            target = m_heap.onlyLocalAllocation(node->child1().node());
+            if (target && target->isRegExpObjectAllocation())
+                exactRead = RegExpObjectLastIndexPLoc;
+            else
+                m_heap.escape(node->child1().node());
+            break;
+
+        case SetRegExpObjectLastIndex:
+            target = m_heap.onlyLocalAllocation(node->child1().node());
+            if (target && target->isRegExpObjectAllocation()) {
+                writes.add(
+                    PromotedLocationDescriptor(RegExpObjectLastIndexPLoc),
+                    LazyNode(node->child2().node()));
+            } else {
+                m_heap.escape(node->child1().node());
+                m_heap.escape(node->child2().node());
+            }
+            break;
+
         case Check:
+        case CheckVarargs:
             m_graph.doToChildren(
                 node,
                 [&] (Edge edge) {
@@ -1046,6 +1088,12 @@ private:
         case PutHint:
             // Handled by OSR availability analysis
             break;
+            
+        case FilterCallLinkStatus:
+        case FilterGetByIdStatus:
+        case FilterPutByIdStatus:
+        case FilterInByIdStatus:
+            break;
 
         default:
             m_graph.doToChildren(
@@ -1061,7 +1109,7 @@ private:
             ASSERT(writes.isEmpty());
             if (Node* value = heapResolve(PromotedHeapLocation(target->identifier(), exactRead))) {
                 ASSERT(!value->replacement());
-                node->replaceWith(value);
+                node->replaceWith(m_graph, value);
             }
             Node* identifier = target->get(exactRead);
             if (identifier)
@@ -1086,6 +1134,7 @@ private:
         m_materializationToEscapee.clear();
         m_materializationSiteToMaterializations.clear();
         m_materializationSiteToRecoveries.clear();
+        m_materializationSiteToHints.clear();
 
         // Logically we wish to consider every allocation and sink
         // it. However, it is probably not profitable to sink an
@@ -1152,17 +1201,17 @@ private:
 
             // The sink candidates are initially the unescaped
             // allocations dying at tail of blocks
-            HashSet<Node*> allocations;
+            NodeSet allocations;
             for (const auto& entry : m_heap.allocations()) {
                 if (!entry.value.isEscapedAllocation())
-                    allocations.add(entry.key);
+                    allocations.addVoid(entry.key);
             }
 
             m_heap.pruneByLiveness(m_combinedLiveness.liveAtTail[block]);
 
             for (Node* identifier : allocations) {
                 if (!m_heap.isAllocation(identifier))
-                    m_sinkCandidates.add(identifier);
+                    m_sinkCandidates.addVoid(identifier);
             }
         }
 
@@ -1180,7 +1229,7 @@ private:
         if (m_sinkCandidates.isEmpty())
             return hasUnescapedReads;
 
-        if (verbose)
+        if (DFGObjectAllocationSinkingPhaseInternal::verbose)
             dataLog("Candidates: ", listDump(m_sinkCandidates), "\n");
 
         // Create the materialization nodes
@@ -1229,28 +1278,43 @@ private:
     {
         // We don't create materializations if the escapee is not a
         // sink candidate
-        Vector<Node*> toRemove;
-        for (const auto& entry : escapees) {
-            if (!m_sinkCandidates.contains(entry.key))
-                toRemove.append(entry.key);
-        }
-        for (Node* identifier : toRemove)
-            escapees.remove(identifier);
-
+        escapees.removeIf(
+            [&] (const auto& entry) {
+                return !m_sinkCandidates.contains(entry.key);
+            });
         if (escapees.isEmpty())
             return;
 
         // First collect the hints that will be needed when the node
-        // we materialize is still stored into other unescaped sink candidates
-        Vector<PromotedHeapLocation> hints;
+        // we materialize is still stored into other unescaped sink candidates.
+        // The way to interpret this vector is:
+        //
+        // PromotedHeapLocation(NotEscapedAllocation, field) = identifierAllocation
+        //
+        // e.g:
+        // PromotedHeapLocation(@PhantomNewFunction, FunctionActivationPLoc) = IdentifierOf(@MaterializeCreateActivation)
+        // or:
+        // PromotedHeapLocation(@PhantomCreateActivation, ClosureVarPLoc(x)) = IdentifierOf(@NewFunction)
+        //
+        // When the rhs of the `=` is to be materialized at this `where` point in the program
+        // and IdentifierOf(Materialization) is the original sunken allocation of the materialization.
+        //
+        // The reason we need to collect all the `identifiers` here is that
+        // we may materialize multiple versions of the allocation along control
+        // flow edges. We will PutHint these values along those edges. However,
+        // we also need to PutHint them when we join and have a Phi of the allocations.
+        Vector<std::pair<PromotedHeapLocation, Node*>> hints;
         for (const auto& entry : m_heap.allocations()) {
             if (escapees.contains(entry.key))
                 continue;
 
             for (const auto& field : entry.value.fields()) {
                 ASSERT(m_sinkCandidates.contains(entry.key) || !escapees.contains(field.value));
-                if (escapees.contains(field.value) && !field.key.neededForMaterialization())
-                    hints.append(PromotedHeapLocation(entry.key, field.key));
+                auto iter = escapees.find(field.value);
+                if (iter != escapees.end()) {
+                    ASSERT(m_sinkCandidates.contains(field.value));
+                    hints.append(std::make_pair(PromotedHeapLocation(entry.key, field.key), field.value));
+                }
             }
         }
 
@@ -1289,28 +1353,28 @@ private:
 
 
         // Compute dependencies between materializations
-        HashMap<Node*, HashSet<Node*>> dependencies;
-        HashMap<Node*, HashSet<Node*>> reverseDependencies;
-        HashMap<Node*, HashSet<Node*>> forMaterialization;
+        HashMap<Node*, NodeSet> dependencies;
+        HashMap<Node*, NodeSet> reverseDependencies;
+        HashMap<Node*, NodeSet> forMaterialization;
         for (const auto& entry : escapees) {
-            auto& myDependencies = dependencies.add(entry.key, HashSet<Node*>()).iterator->value;
-            auto& myDependenciesForMaterialization = forMaterialization.add(entry.key, HashSet<Node*>()).iterator->value;
-            reverseDependencies.add(entry.key, HashSet<Node*>());
+            auto& myDependencies = dependencies.add(entry.key, NodeSet()).iterator->value;
+            auto& myDependenciesForMaterialization = forMaterialization.add(entry.key, NodeSet()).iterator->value;
+            reverseDependencies.add(entry.key, NodeSet());
             for (const auto& field : entry.value.fields()) {
                 if (escapees.contains(field.value) && field.value != entry.key) {
-                    myDependencies.add(field.value);
-                    reverseDependencies.add(field.value, HashSet<Node*>()).iterator->value.add(entry.key);
+                    myDependencies.addVoid(field.value);
+                    reverseDependencies.add(field.value, NodeSet()).iterator->value.addVoid(entry.key);
                     if (field.key.neededForMaterialization())
-                        myDependenciesForMaterialization.add(field.value);
+                        myDependenciesForMaterialization.addVoid(field.value);
                 }
             }
         }
 
         // Helper function to update the materialized set and the
         // dependencies
-        HashSet<Node*> materialized;
+        NodeSet materialized;
         auto materialize = [&] (Node* identifier) {
-            materialized.add(identifier);
+            materialized.addVoid(identifier);
             for (Node* dep : dependencies.get(identifier))
                 reverseDependencies.find(dep)->value.remove(identifier);
             for (Node* rdep : reverseDependencies.get(identifier)) {
@@ -1325,7 +1389,7 @@ private:
         // Nodes without remaining unmaterialized fields will be
         // materialized first - amongst the remaining unmaterialized
         // nodes
-        std::list<Allocation> toMaterialize;
+        StdList<Allocation> toMaterialize;
         auto firstPos = toMaterialize.begin();
         auto materializeFirst = [&] (Allocation&& allocation) {
             materialize(allocation.identifier());
@@ -1378,7 +1442,7 @@ private:
             // first.
             if (materialized.isEmpty()) {
                 uint64_t maxEvaluation = 0;
-                Allocation* bestAllocation;
+                Allocation* bestAllocation = nullptr;
                 for (auto& entry : escapees) {
                     if (!forMaterialization.find(entry.key)->value.isEmpty())
                         continue;
@@ -1402,15 +1466,15 @@ private:
 
         materialized.clear();
 
-        HashSet<Node*> escaped;
+        NodeSet escaped;
         for (const Allocation& allocation : toMaterialize)
-            escaped.add(allocation.identifier());
+            escaped.addVoid(allocation.identifier());
         for (const Allocation& allocation : toMaterialize) {
             for (const auto& field : allocation.fields()) {
                 if (escaped.contains(field.value) && !materialized.contains(field.value))
                     toRecover.append(PromotedHeapLocation(allocation.identifier(), field.key));
             }
-            materialized.add(allocation.identifier());
+            materialized.addVoid(allocation.identifier());
         }
 
         Vector<Node*>& materializations = m_materializationSiteToMaterializations.add(
@@ -1429,10 +1493,8 @@ private:
 
         // The hints need to be after the "real" recoveries so that we
         // don't hint not-yet-complete objects
-        if (!hints.isEmpty()) {
-            m_materializationSiteToRecoveries.add(
-                where, Vector<PromotedHeapLocation>()).iterator->value.appendVector(hints);
-        }
+        m_materializationSiteToHints.add(
+            where, Vector<std::pair<PromotedHeapLocation, Node*>>()).iterator->value.appendVector(hints);
     }
 
     Node* createMaterialization(const Allocation& allocation, Node* where)
@@ -1443,29 +1505,39 @@ private:
         switch (allocation.kind()) {
         case Allocation::Kind::Object: {
             ObjectMaterializationData* data = m_graph.m_objectMaterializationData.add();
-            StructureSet* set = m_graph.addStructureSet(allocation.structures());
 
             return m_graph.addNode(
                 allocation.identifier()->prediction(), Node::VarArg, MaterializeNewObject,
                 where->origin.withSemantic(allocation.identifier()->origin.semantic),
-                OpInfo(set), OpInfo(data), 0, 0);
+                OpInfo(m_graph.addStructureSet(allocation.structures())), OpInfo(data), 0, 0);
         }
 
-        case Allocation::Kind::ArrowFunction:
+        case Allocation::Kind::AsyncGeneratorFunction:
+        case Allocation::Kind::AsyncFunction:
         case Allocation::Kind::GeneratorFunction:
         case Allocation::Kind::Function: {
             FrozenValue* executable = allocation.identifier()->cellOperand();
             
-            NodeType nodeType =
-                allocation.kind() == Allocation::Kind::ArrowFunction ? NewArrowFunction :
-                allocation.kind() == Allocation::Kind::GeneratorFunction ? NewGeneratorFunction : NewFunction;
-            
+            NodeType nodeType;
+            switch (allocation.kind()) {
+            case Allocation::Kind::GeneratorFunction:
+                nodeType = NewGeneratorFunction;
+                break;
+            case Allocation::Kind::AsyncGeneratorFunction:
+                nodeType = NewAsyncGeneratorFunction;
+                break;
+            case Allocation::Kind::AsyncFunction:
+                nodeType = NewAsyncFunction;
+                break;
+            default:
+                nodeType = NewFunction;
+            }
+
             return m_graph.addNode(
                 allocation.identifier()->prediction(), nodeType,
                 where->origin.withSemantic(
                     allocation.identifier()->origin.semantic),
                 OpInfo(executable));
-            break;
         }
 
         case Allocation::Kind::Activation: {
@@ -1479,6 +1551,15 @@ private:
                 OpInfo(symbolTable), OpInfo(data), 0, 0);
         }
 
+        case Allocation::Kind::RegExpObject: {
+            FrozenValue* regExp = allocation.identifier()->cellOperand();
+            return m_graph.addNode(
+                allocation.identifier()->prediction(), NewRegexp,
+                where->origin.withSemantic(
+                    allocation.identifier()->origin.semantic),
+                OpInfo(regExp));
+        }
+
         default:
             DFG_CRASH(m_graph, allocation.identifier(), "Bad allocation kind");
         }
@@ -1499,10 +1580,10 @@ private:
                         // If the location is not on a sink candidate,
                         // we only sink it if it is read
                         if (m_sinkCandidates.contains(location.base()))
-                            locations.add(location);
+                            locations.addVoid(location);
                     },
                     [&] (PromotedHeapLocation location) -> Node* {
-                        locations.add(location);
+                        locations.addVoid(location);
                         return nullptr;
                     });
             }
@@ -1547,6 +1628,9 @@ private:
         HashMap<FrozenValue*, Node*> lazyMapping;
         if (!m_bottom)
             m_bottom = m_insertionSet.insertConstant(0, m_graph.block(0)->at(0)->origin, jsNumber(1927));
+
+        Vector<HashSet<PromotedHeapLocation>> hintsForPhi(m_sinkCandidates.size());
+
         for (BasicBlock* block : m_graph.blocksInNaturalOrder()) {
             m_heap = m_heapAtHead[block];
 
@@ -1568,6 +1652,31 @@ private:
                     m_allocationSSA.newDef(m_nodeToVariable.get(escapee), block, materialization);
                 }
 
+                for (std::pair<PromotedHeapLocation, Node*> pair : m_materializationSiteToHints.get(node)) {
+                    PromotedHeapLocation location = pair.first;
+                    Node* identifier = pair.second;
+                    // We're materializing `identifier` at this point, and the unmaterialized
+                    // version is inside `location`. We track which SSA variable this belongs
+                    // to in case we also need a PutHint for the Phi.
+                    if (UNLIKELY(validationEnabled())) {
+                        RELEASE_ASSERT(m_sinkCandidates.contains(location.base()));
+                        RELEASE_ASSERT(m_sinkCandidates.contains(identifier));
+
+                        bool found = false;
+                        for (Node* materialization : m_materializationSiteToMaterializations.get(node)) {
+                            // We're materializing `identifier` here. This asserts that this is indeed the case.
+                            if (m_materializationToEscapee.get(materialization) == identifier) {
+                                found = true;
+                                break;
+                            }
+                        }
+                        RELEASE_ASSERT(found);
+                    }
+
+                    SSACalculator::Variable* variable = m_nodeToVariable.get(identifier);
+                    hintsForPhi[variable->index()].addVoid(location);
+                }
+
                 if (m_sinkCandidates.contains(node))
                     m_allocationSSA.newDef(m_nodeToVariable.get(node), block, node);
 
@@ -1643,7 +1752,7 @@ private:
 
         // Place Phis in the right places, replace all uses of any load with the appropriate
         // value, and create the materialization nodes.
-        LocalOSRAvailabilityCalculator availabilityCalculator;
+        LocalOSRAvailabilityCalculator availabilityCalculator(m_graph);
         m_graph.clearReplacements();
         for (BasicBlock* block : m_graph.blocksInPreOrder()) {
             m_heap = m_heapAtHead[block];
@@ -1683,9 +1792,15 @@ private:
                 insertOSRHintsForUpdate(
                     0, block->at(0)->origin, canExit,
                     availabilityCalculator.m_availability, identifier, phiDef->value());
+
+                for (PromotedHeapLocation location : hintsForPhi[variable->index()]) {
+                    m_insertionSet.insert(0,
+                        location.createHint(m_graph, block->at(0)->origin.withInvalidExit(), phiDef->value()));
+                    m_localMapping.set(location, phiDef->value());
+                }
             }
 
-            if (verbose) {
+            if (DFGObjectAllocationSinkingPhaseInternal::verbose) {
                 dataLog("Local mapping at ", pointerDump(block), ": ", mapDump(m_localMapping), "\n");
                 dataLog("Local materializations at ", pointerDump(block), ": ", mapDump(m_escapeeToMaterialization), "\n");
             }
@@ -1701,6 +1816,8 @@ private:
                     m_localMapping.set(location, m_bottom);
 
                     if (m_sinkCandidates.contains(node)) {
+                        if (DFGObjectAllocationSinkingPhaseInternal::verbose)
+                            dataLog("For sink candidate ", node, " found location ", location, "\n");
                         m_insertionSet.insert(
                             nodeIndex + 1,
                             location.createHint(
@@ -1714,12 +1831,14 @@ private:
                     populateMaterialization(block, materialization, escapee);
                     m_escapeeToMaterialization.set(escapee, materialization);
                     m_insertionSet.insert(nodeIndex, materialization);
-                    if (verbose)
+                    if (DFGObjectAllocationSinkingPhaseInternal::verbose)
                         dataLog("Materializing ", escapee, " => ", materialization, " at ", node, "\n");
                 }
 
                 for (PromotedHeapLocation location : m_materializationSiteToRecoveries.get(node))
                     m_insertionSet.insert(nodeIndex, createRecovery(block, location, node, canExit));
+                for (std::pair<PromotedHeapLocation, Node*> pair : m_materializationSiteToHints.get(node))
+                    m_insertionSet.insert(nodeIndex, createRecovery(block, pair.first, node, canExit));
 
                 // We need to put the OSR hints after the recoveries,
                 // because we only want the hints once the object is
@@ -1767,6 +1886,8 @@ private:
 
                         doLower = true;
 
+                        if (DFGObjectAllocationSinkingPhaseInternal::verbose)
+                            dataLog("Creating hint with value ", nodeValue, " before ", node, "\n");
                         m_insertionSet.insert(
                             nodeIndex + 1,
                             location.createHint(
@@ -1788,7 +1909,6 @@ private:
                         node->convertToPhantomNewObject();
                         break;
 
-                    case NewArrowFunction:
                     case NewFunction:
                         node->convertToPhantomNewFunction();
                         break;
@@ -1797,12 +1917,24 @@ private:
                         node->convertToPhantomNewGeneratorFunction();
                         break;
 
+                    case NewAsyncGeneratorFunction:
+                        node->convertToPhantomNewAsyncGeneratorFunction();
+                        break;
+
+                    case NewAsyncFunction:
+                        node->convertToPhantomNewAsyncFunction();
+                        break;
+
                     case CreateActivation:
                         node->convertToPhantomCreateActivation();
                         break;
 
+                    case NewRegexp:
+                        node->convertToPhantomNewRegexp();
+                        break;
+
                     default:
-                        node->remove();
+                        node->remove(m_graph);
                         break;
                     }
                 }
@@ -1845,7 +1977,7 @@ private:
         }
     }
 
-    Node* resolve(BasicBlock* block, PromotedHeapLocation location)
+    NEVER_INLINE Node* resolve(BasicBlock* block, PromotedHeapLocation location)
     {
         // If we are currently pointing to a single local allocation,
         // simply return the associated materialization.
@@ -1862,14 +1994,15 @@ private:
         ASSERT(def->value());
 
         Node* result = def->value();
-
+        if (result->replacement())
+            result = result->replacement();
         ASSERT(!result->replacement());
 
         m_localMapping.add(location, result);
         return result;
     }
 
-    Node* resolve(BasicBlock* block, Node* node)
+    NEVER_INLINE Node* resolve(BasicBlock* block, Node* node)
     {
         // If we are currently pointing to a single local allocation,
         // simply return the associated materialization.
@@ -1883,7 +2016,7 @@ private:
         return node;
     }
 
-    Node* getMaterialization(BasicBlock* block, Node* identifier)
+    NEVER_INLINE Node* getMaterialization(BasicBlock* block, Node* identifier)
     {
         ASSERT(m_heap.isAllocation(identifier));
         if (!m_sinkCandidates.contains(identifier))
@@ -1902,6 +2035,13 @@ private:
 
     void insertOSRHintsForUpdate(unsigned nodeIndex, NodeOrigin origin, bool& canExit, AvailabilityMap& availability, Node* escapee, Node* materialization)
     {
+        if (DFGObjectAllocationSinkingPhaseInternal::verbose) {
+            dataLog("Inserting OSR hints at ", origin, ":\n");
+            dataLog("    Escapee: ", escapee, "\n");
+            dataLog("    Materialization: ", materialization, "\n");
+            dataLog("    Availability: ", availability, "\n");
+        }
+        
         // We need to follow() the value in the heap.
         // Consider the following graph:
         //
@@ -1974,7 +2114,7 @@ private:
 
                 case NamedPropertyPLoc: {
                     ASSERT(location.base() == allocation.identifier());
-                    data.m_properties.append(PhantomPropertyValue(location.info()));
+                    data.m_properties.append(location.descriptor());
                     Node* value = resolve(block, location);
                     if (m_sinkCandidates.contains(value))
                         m_graph.m_varArgChildren.append(m_bottom);
@@ -2024,7 +2164,7 @@ private:
 
                 case ClosureVarPLoc: {
                     ASSERT(location.base() == allocation.identifier());
-                    data.m_properties.append(PhantomPropertyValue(location.info()));
+                    data.m_properties.append(location.descriptor());
                     Node* value = resolve(block, location);
                     if (m_sinkCandidates.contains(value))
                         m_graph.m_varArgChildren.append(m_bottom);
@@ -2045,8 +2185,9 @@ private:
         }
         
         case NewFunction:
-        case NewArrowFunction:
-        case NewGeneratorFunction: {
+        case NewGeneratorFunction:
+        case NewAsyncGeneratorFunction:
+        case NewAsyncFunction: {
             Vector<PromotedHeapLocation> locations = m_locationsForAllocation.get(escapee);
             ASSERT(locations.size() == 2);
                 
@@ -2060,6 +2201,23 @@ private:
             break;
         }
 
+        case NewRegexp: {
+            Vector<PromotedHeapLocation> locations = m_locationsForAllocation.get(escapee);
+            ASSERT(locations.size() == 2);
+
+            PromotedHeapLocation regExp(RegExpObjectRegExpPLoc, allocation.identifier());
+            ASSERT_UNUSED(regExp, locations.contains(regExp));
+
+            PromotedHeapLocation lastIndex(RegExpObjectLastIndexPLoc, allocation.identifier());
+            ASSERT(locations.contains(lastIndex));
+            Node* value = resolve(block, lastIndex);
+            if (m_sinkCandidates.contains(value))
+                node->child1() = Edge(m_bottom);
+            else
+                node->child1() = Edge(value);
+            break;
+        }
+
         default:
             DFG_CRASH(m_graph, node, "Bad materialize op");
         }
@@ -2067,7 +2225,7 @@ private:
 
     Node* createRecovery(BasicBlock* block, PromotedHeapLocation location, Node* where, bool& canExit)
     {
-        if (verbose)
+        if (DFGObjectAllocationSinkingPhaseInternal::verbose)
             dataLog("Recovering ", location, " at ", where, "\n");
         ASSERT(location.base()->isPhantomAllocation());
         Node* base = getMaterialization(block, location.base());
@@ -2075,7 +2233,7 @@ private:
 
         NodeOrigin origin = where->origin.withSemantic(base->origin.semantic);
 
-        if (verbose)
+        if (DFGObjectAllocationSinkingPhaseInternal::verbose)
             dataLog("Base is ", base, " and value is ", value, "\n");
 
         if (base->isPhantomAllocation()) {
@@ -2087,7 +2245,7 @@ private:
         case NamedPropertyPLoc: {
             Allocation& allocation = m_heap.getAllocation(location.base());
 
-            Vector<Structure*> structures;
+            Vector<RegisteredStructure> structures;
             structures.appendRange(allocation.structures().begin(), allocation.structures().end());
             unsigned identifierNumber = location.info();
             UniquedStringImpl* uid = m_graph.identifiers()[identifierNumber];
@@ -2095,10 +2253,11 @@ private:
             std::sort(
                 structures.begin(),
                 structures.end(),
-                [uid] (Structure *a, Structure* b) -> bool {
+                [uid] (RegisteredStructure a, RegisteredStructure b) -> bool {
                     return a->getConcurrently(uid) < b->getConcurrently(uid);
                 });
 
+            RELEASE_ASSERT(structures.size());
             PropertyOffset firstOffset = structures[0]->getConcurrently(uid);
 
             if (firstOffset == structures.last()->getConcurrently(uid)) {
@@ -2112,7 +2271,6 @@ private:
                 data->identifierNumber = identifierNumber;
 
                 return m_graph.addNode(
-                    SpecNone,
                     PutByOffset,
                     origin.takeValidExit(canExit),
                     OpInfo(data),
@@ -2127,7 +2285,7 @@ private:
             {
                 PropertyOffset currentOffset = firstOffset;
                 StructureSet currentSet;
-                for (Structure* structure : structures) {
+                for (RegisteredStructure structure : structures) {
                     PropertyOffset offset = structure->getConcurrently(uid);
                     if (offset != currentOffset) {
                         // Because our analysis treats MultiPutByOffset like an escape, we only have to
@@ -2136,41 +2294,67 @@ private:
                         // at this point, we can simply trust that the incoming value has the right type
                         // for whatever structure we are using.
                         data->variants.append(
-                            PutByIdVariant::replace(currentSet, currentOffset, InferredType::Top));
+                            PutByIdVariant::replace(currentSet, currentOffset));
                         currentOffset = offset;
                         currentSet.clear();
                     }
-                    currentSet.add(structure);
+                    currentSet.add(structure.get());
                 }
                 data->variants.append(
-                    PutByIdVariant::replace(currentSet, currentOffset, InferredType::Top));
+                    PutByIdVariant::replace(currentSet, currentOffset));
             }
 
             return m_graph.addNode(
-                SpecNone,
                 MultiPutByOffset,
                 origin.takeValidExit(canExit),
                 OpInfo(data),
                 Edge(base, KnownCellUse),
                 value->defaultEdge());
-            break;
         }
 
         case ClosureVarPLoc: {
             return m_graph.addNode(
-                SpecNone,
                 PutClosureVar,
                 origin.takeValidExit(canExit),
                 OpInfo(location.info()),
                 Edge(base, KnownCellUse),
                 value->defaultEdge());
-            break;
+        }
+
+        case RegExpObjectLastIndexPLoc: {
+            return m_graph.addNode(
+                SetRegExpObjectLastIndex,
+                origin.takeValidExit(canExit),
+                OpInfo(true),
+                Edge(base, KnownCellUse),
+                value->defaultEdge());
         }
 
         default:
             DFG_CRASH(m_graph, base, "Bad location kind");
             break;
         }
+
+        RELEASE_ASSERT_NOT_REACHED();
+    }
+    
+    void removeICStatusFilters()
+    {
+        for (BasicBlock* block : m_graph.blocksInNaturalOrder()) {
+            for (Node* node : *block) {
+                switch (node->op()) {
+                case FilterCallLinkStatus:
+                case FilterGetByIdStatus:
+                case FilterPutByIdStatus:
+                case FilterInByIdStatus:
+                    if (node->child1()->isPhantomAllocation())
+                        node->removeWithoutChecks();
+                    break;
+                default:
+                    break;
+                }
+            }
+        }
     }
 
     // This is a great way of asking value->isStillValid() without having to worry about getting
@@ -2184,7 +2368,7 @@ private:
 
     SSACalculator m_pointerSSA;
     SSACalculator m_allocationSSA;
-    HashSet<Node*> m_sinkCandidates;
+    NodeSet m_sinkCandidates;
     HashMap<PromotedHeapLocation, SSACalculator::Variable*> m_locationToVariable;
     HashMap<Node*, SSACalculator::Variable*> m_nodeToVariable;
     HashMap<PromotedHeapLocation, Node*> m_localMapping;
@@ -2197,6 +2381,7 @@ private:
     HashMap<Node*, Node*> m_materializationToEscapee;
     HashMap<Node*, Vector<Node*>> m_materializationSiteToMaterializations;
     HashMap<Node*, Vector<PromotedHeapLocation>> m_materializationSiteToRecoveries;
+    HashMap<Node*, Vector<std::pair<PromotedHeapLocation, Node*>>> m_materializationSiteToHints;
 
     HashMap<Node*, Vector<PromotedHeapLocation>> m_locationsForAllocation;