[JSC] Store bits for JSRopeString in 3 stores
[WebKit-https.git] / Source / JavaScriptCore / runtime / JSString.cpp
index d31ca6b..c836fa2 100644 (file)
@@ -40,15 +40,28 @@ Structure* JSString::createStructure(VM& vm, JSGlobalObject* globalObject, JSVal
     return Structure::create(vm, globalObject, proto, TypeInfo(StringType, StructureFlags), info());
 }
 
+JSString* JSString::createEmptyString(VM& vm)
+{
+    JSString* newString = new (NotNull, allocateCell<JSString>(vm.heap)) JSString(vm, *StringImpl::empty());
+    newString->finishCreation(vm);
+    return newString;
+}
+
 template<>
 void JSRopeString::RopeBuilder<RecordOverflow>::expand()
 {
     RELEASE_ASSERT(!this->hasOverflowed());
-    ASSERT(m_index == JSRopeString::s_maxInternalRopeLength);
-    JSString* jsString = m_jsString;
-    m_jsString = jsStringBuilder(&m_vm);
-    m_index = 0;
-    append(jsString);
+    ASSERT(m_strings.size() == JSRopeString::s_maxInternalRopeLength);
+    static_assert(3 == JSRopeString::s_maxInternalRopeLength, "");
+    ASSERT(m_length);
+    ASSERT(asString(m_strings.at(0))->length());
+    ASSERT(asString(m_strings.at(1))->length());
+    ASSERT(asString(m_strings.at(2))->length());
+
+    JSString* string = JSRopeString::create(m_vm, asString(m_strings.at(0)), asString(m_strings.at(1)), asString(m_strings.at(2)));
+    ASSERT(string->length() == m_length);
+    m_strings.clear();
+    m_strings.append(string);
 }
 
 void JSString::destroy(JSCell* cell)
@@ -61,14 +74,16 @@ void JSString::dumpToStream(const JSCell* cell, PrintStream& out)
     VM& vm = *cell->vm();
     const JSString* thisObject = jsCast<const JSString*>(cell);
     out.printf("<%p, %s, [%u], ", thisObject, thisObject->className(vm), thisObject->length());
-    if (thisObject->isRope())
+    uintptr_t pointer = thisObject->m_fiber;
+    if (pointer & isRopeInPointer)
         out.printf("[rope]");
     else {
-        WTF::StringImpl* ourImpl = thisObject->m_value.impl();
-        if (ourImpl->is8Bit())
-            out.printf("[8 %p]", ourImpl->characters8());
-        else
-            out.printf("[16 %p]", ourImpl->characters16());
+        if (WTF::StringImpl* ourImpl = bitwise_cast<StringImpl*>(pointer)) {
+            if (ourImpl->is8Bit())
+                out.printf("[8 %p]", ourImpl->characters8());
+            else
+                out.printf("[16 %p]", ourImpl->characters16());
+        }
     }
     out.printf(">");
 }
@@ -86,9 +101,10 @@ bool JSString::equalSlowCase(ExecState* exec, JSString* other) const
 size_t JSString::estimatedSize(JSCell* cell, VM& vm)
 {
     JSString* thisObject = asString(cell);
-    if (thisObject->isRope())
+    uintptr_t pointer = thisObject->m_fiber;
+    if (pointer & isRopeInPointer)
         return Base::estimatedSize(cell, vm);
-    return Base::estimatedSize(cell, vm) + thisObject->m_value.impl()->costDuringGC();
+    return Base::estimatedSize(cell, vm) + bitwise_cast<StringImpl*>(pointer)->costDuringGC();
 }
 
 void JSString::visitChildren(JSCell* cell, SlotVisitor& visitor)
@@ -96,20 +112,36 @@ void JSString::visitChildren(JSCell* cell, SlotVisitor& visitor)
     JSString* thisObject = asString(cell);
     Base::visitChildren(thisObject, visitor);
     
-    if (thisObject->isRope())
-        static_cast<JSRopeString*>(thisObject)->visitFibers(visitor);
-    if (StringImpl* impl = thisObject->m_value.impl())
-        visitor.reportExtraMemoryVisited(impl->costDuringGC());
-}
-
-void JSRopeString::visitFibers(SlotVisitor& visitor)
-{
-    if (isSubstring()) {
-        visitor.append(substringBase());
+    uintptr_t pointer = thisObject->m_fiber;
+    if (pointer & isRopeInPointer) {
+        if (pointer & JSRopeString::isSubstringInPointer) {
+            visitor.appendUnbarriered(static_cast<JSRopeString*>(thisObject)->fiber1());
+            return;
+        }
+        for (unsigned index = 0; index < JSRopeString::s_maxInternalRopeLength; ++index) {
+            JSString* fiber = nullptr;
+            switch (index) {
+            case 0:
+                fiber = bitwise_cast<JSString*>(pointer & JSRopeString::stringMask);
+                break;
+            case 1:
+                fiber = static_cast<JSRopeString*>(thisObject)->fiber1();
+                break;
+            case 2:
+                fiber = static_cast<JSRopeString*>(thisObject)->fiber2();
+                break;
+            default:
+                ASSERT_NOT_REACHED();
+                return;
+            }
+            if (!fiber)
+                break;
+            visitor.appendUnbarriered(fiber);
+        }
         return;
     }
-    for (size_t i = 0; i < s_maxInternalRopeLength && fiber(i); ++i)
-        visitor.append(fiber(i));
+    if (StringImpl* impl = bitwise_cast<StringImpl*>(pointer))
+        visitor.reportExtraMemoryVisited(impl->costDuringGC());
 }
 
 static const unsigned maxLengthForOnStackResolve = 2048;
@@ -117,7 +149,7 @@ static const unsigned maxLengthForOnStackResolve = 2048;
 void JSRopeString::resolveRopeInternal8(LChar* buffer) const
 {
     if (isSubstring()) {
-        StringImpl::copyCharacters(buffer, substringBase()->m_value.characters8() + substringOffset(), length());
+        StringImpl::copyCharacters(buffer, substringBase()->valueInternal().characters8() + substringOffset(), length());
         return;
     }
     
@@ -135,7 +167,7 @@ void JSRopeString::resolveRopeInternal8NoSubstring(LChar* buffer) const
 
     LChar* position = buffer;
     for (size_t i = 0; i < s_maxInternalRopeLength && fiber(i); ++i) {
-        const StringImpl& fiberString = *fiber(i)->m_value.impl();
+        const StringImpl& fiberString = *fiber(i)->valueInternal().impl();
         unsigned length = fiberString.length();
         StringImpl::copyCharacters(position, fiberString.characters8(), length);
         position += length;
@@ -147,7 +179,7 @@ void JSRopeString::resolveRopeInternal16(UChar* buffer) const
 {
     if (isSubstring()) {
         StringImpl::copyCharacters(
-            buffer, substringBase()->m_value.characters16() + substringOffset(), length());
+            buffer, substringBase()->valueInternal().characters16() + substringOffset(), length());
         return;
     }
     
@@ -165,7 +197,7 @@ void JSRopeString::resolveRopeInternal16NoSubstring(UChar* buffer) const
 
     UChar* position = buffer;
     for (size_t i = 0; i < s_maxInternalRopeLength && fiber(i); ++i) {
-        const StringImpl& fiberString = *fiber(i)->m_value.impl();
+        const StringImpl& fiberString = *fiber(i)->valueInternal().impl();
         unsigned length = fiberString.length();
         if (fiberString.is8Bit())
             StringImpl::copyCharacters(position, fiberString.characters8(), length);
@@ -176,73 +208,75 @@ void JSRopeString::resolveRopeInternal16NoSubstring(UChar* buffer) const
     ASSERT((buffer + length()) == position);
 }
 
-void JSRopeString::resolveRopeToAtomicString(ExecState* exec) const
+AtomicString JSRopeString::resolveRopeToAtomicString(ExecState* exec) const
 {
     VM& vm = exec->vm();
     auto scope = DECLARE_THROW_SCOPE(vm);
 
     if (length() > maxLengthForOnStackResolve) {
-        resolveRope(exec);
-        RETURN_IF_EXCEPTION(scope, void());
-        m_value = AtomicString(m_value);
-        setIs8Bit(m_value.impl()->is8Bit());
-        return;
+        scope.release();
+        return resolveRopeWithFunction(exec, [&] (Ref<StringImpl>&& newImpl) {
+            return AtomicStringImpl::add(newImpl.ptr());
+        });
     }
 
     if (is8Bit()) {
         LChar buffer[maxLengthForOnStackResolve];
         resolveRopeInternal8(buffer);
-        m_value = AtomicString(buffer, length());
-        setIs8Bit(m_value.impl()->is8Bit());
+        convertToNonRope(AtomicStringImpl::add(buffer, length()));
     } else {
         UChar buffer[maxLengthForOnStackResolve];
         resolveRopeInternal16(buffer);
-        m_value = AtomicString(buffer, length());
-        setIs8Bit(m_value.impl()->is8Bit());
+        convertToNonRope(AtomicStringImpl::add(buffer, length()));
     }
 
-    clearFibers();
-
     // If we resolved a string that didn't previously exist, notify the heap that we've grown.
-    if (m_value.impl()->hasOneRef())
-        vm.heap.reportExtraMemoryAllocated(m_value.impl()->cost());
+    if (valueInternal().impl()->hasOneRef())
+        vm.heap.reportExtraMemoryAllocated(valueInternal().impl()->cost());
+    return valueInternal();
 }
 
-void JSRopeString::clearFibers() const
+inline void JSRopeString::convertToNonRope(String&& string) const
 {
-    for (size_t i = 0; i < s_maxInternalRopeLength; ++i)
-        u[i].number = 0;
+    // Concurrent compiler threads can access String held by JSString. So we always emit
+    // store-store barrier here to ensure concurrent compiler threads see initialized String.
+    ASSERT(JSString::isRope());
+    WTF::storeStoreFence();
+    new (&uninitializedValueInternal()) String(WTFMove(string));
+    static_assert(sizeof(String) == sizeof(RefPtr<StringImpl>), "JSString's String initialization must be done in one pointer move.");
+    // We do not clear the trailing fibers and length information (fiber1 and fiber2) because we could be reading the length concurrently.
+    ASSERT(!JSString::isRope());
 }
 
 RefPtr<AtomicStringImpl> JSRopeString::resolveRopeToExistingAtomicString(ExecState* exec) const
 {
+    VM& vm = exec->vm();
+    auto scope = DECLARE_THROW_SCOPE(vm);
+
     if (length() > maxLengthForOnStackResolve) {
-        resolveRope(exec);
-        if (RefPtr<AtomicStringImpl> existingAtomicString = AtomicStringImpl::lookUp(m_value.impl())) {
-            m_value = *existingAtomicString;
-            setIs8Bit(m_value.impl()->is8Bit());
-            clearFibers();
-            return existingAtomicString;
-        }
-        return nullptr;
+        RefPtr<AtomicStringImpl> existingAtomicString;
+        resolveRopeWithFunction(exec, [&] (Ref<StringImpl>&& newImpl) -> Ref<StringImpl> {
+            existingAtomicString = AtomicStringImpl::lookUp(newImpl.ptr());
+            if (existingAtomicString)
+                return makeRef(*existingAtomicString);
+            return WTFMove(newImpl);
+        });
+        RETURN_IF_EXCEPTION(scope, nullptr);
+        return existingAtomicString;
     }
     
     if (is8Bit()) {
         LChar buffer[maxLengthForOnStackResolve];
         resolveRopeInternal8(buffer);
         if (RefPtr<AtomicStringImpl> existingAtomicString = AtomicStringImpl::lookUp(buffer, length())) {
-            m_value = *existingAtomicString;
-            setIs8Bit(m_value.impl()->is8Bit());
-            clearFibers();
+            convertToNonRope(*existingAtomicString);
             return existingAtomicString;
         }
     } else {
         UChar buffer[maxLengthForOnStackResolve];
         resolveRopeInternal16(buffer);
         if (RefPtr<AtomicStringImpl> existingAtomicString = AtomicStringImpl::lookUp(buffer, length())) {
-            m_value = *existingAtomicString;
-            setIs8Bit(m_value.impl()->is8Bit());
-            clearFibers();
+            convertToNonRope(*existingAtomicString);
             return existingAtomicString;
         }
     }
@@ -250,44 +284,51 @@ RefPtr<AtomicStringImpl> JSRopeString::resolveRopeToExistingAtomicString(ExecSta
     return nullptr;
 }
 
-void JSRopeString::resolveRope(ExecState* exec) const
+template<typename Function>
+const String& JSRopeString::resolveRopeWithFunction(ExecState* nullOrExecForOOM, Function&& function) const
 {
     ASSERT(isRope());
     
+    VM& vm = *this->vm();
     if (isSubstring()) {
         ASSERT(!substringBase()->isRope());
-        m_value = substringBase()->m_value.substringSharingImpl(substringOffset(), length());
-        substringBase().clear();
-        return;
+        auto newImpl = substringBase()->valueInternal().substringSharingImpl(substringOffset(), length());
+        convertToNonRope(function(newImpl.releaseImpl().releaseNonNull()));
+        return valueInternal();
     }
     
     if (is8Bit()) {
         LChar* buffer;
-        if (auto newImpl = StringImpl::tryCreateUninitialized(length(), buffer)) {
-            exec->vm().heap.reportExtraMemoryAllocated(newImpl->cost());
-            m_value = WTFMove(newImpl);
-        } else {
-            outOfMemory(exec);
-            return;
+        auto newImpl = StringImpl::tryCreateUninitialized(length(), buffer);
+        if (!newImpl) {
+            outOfMemory(nullOrExecForOOM);
+            return nullString();
         }
+        vm.heap.reportExtraMemoryAllocated(newImpl->cost());
+
         resolveRopeInternal8NoSubstring(buffer);
-        clearFibers();
-        ASSERT(!isRope());
-        return;
+        convertToNonRope(function(newImpl.releaseNonNull()));
+        return valueInternal();
     }
     
     UChar* buffer;
-    if (auto newImpl = StringImpl::tryCreateUninitialized(length(), buffer)) {
-        exec->vm().heap.reportExtraMemoryAllocated(newImpl->cost());
-        m_value = WTFMove(newImpl);
-    } else {
-        outOfMemory(exec);
-        return;
+    auto newImpl = StringImpl::tryCreateUninitialized(length(), buffer);
+    if (!newImpl) {
+        outOfMemory(nullOrExecForOOM);
+        return nullString();
     }
+    vm.heap.reportExtraMemoryAllocated(newImpl->cost());
     
     resolveRopeInternal16NoSubstring(buffer);
-    clearFibers();
-    ASSERT(!isRope());
+    convertToNonRope(function(newImpl.releaseNonNull()));
+    return valueInternal();
+}
+
+const String& JSRopeString::resolveRope(ExecState* nullOrExecForOOM) const
+{
+    return resolveRopeWithFunction(nullOrExecForOOM, [] (Ref<StringImpl>&& newImpl) {
+        return WTFMove(newImpl);
+    });
 }
 
 // Overview: These functions convert a JSString from holding a string in rope form
@@ -306,7 +347,7 @@ void JSRopeString::resolveRopeSlowCase8(LChar* buffer) const
     Vector<JSString*, 32, UnsafeVectorOverflow> workQueue; // Putting strings into a Vector is only OK because there are no GC points in this method.
     
     for (size_t i = 0; i < s_maxInternalRopeLength && fiber(i); ++i)
-        workQueue.append(fiber(i).get());
+        workQueue.append(fiber(i));
 
     while (!workQueue.isEmpty()) {
         JSString* currentFiber = workQueue.last();
@@ -318,15 +359,15 @@ void JSRopeString::resolveRopeSlowCase8(LChar* buffer) const
             JSRopeString* currentFiberAsRope = static_cast<JSRopeString*>(currentFiber);
             if (!currentFiberAsRope->isSubstring()) {
                 for (size_t i = 0; i < s_maxInternalRopeLength && currentFiberAsRope->fiber(i); ++i)
-                    workQueue.append(currentFiberAsRope->fiber(i).get());
+                    workQueue.append(currentFiberAsRope->fiber(i));
                 continue;
             }
             ASSERT(!currentFiberAsRope->substringBase()->isRope());
             characters =
-                currentFiberAsRope->substringBase()->m_value.characters8() +
+                currentFiberAsRope->substringBase()->valueInternal().characters8() +
                 currentFiberAsRope->substringOffset();
         } else
-            characters = currentFiber->m_value.characters8();
+            characters = currentFiber->valueInternal().characters8();
         
         unsigned length = currentFiber->length();
         position -= length;
@@ -342,7 +383,7 @@ void JSRopeString::resolveRopeSlowCase(UChar* buffer) const
     Vector<JSString*, 32, UnsafeVectorOverflow> workQueue; // These strings are kept alive by the parent rope, so using a Vector is OK.
 
     for (size_t i = 0; i < s_maxInternalRopeLength && fiber(i); ++i)
-        workQueue.append(fiber(i).get());
+        workQueue.append(fiber(i));
 
     while (!workQueue.isEmpty()) {
         JSString* currentFiber = workQueue.last();
@@ -353,7 +394,7 @@ void JSRopeString::resolveRopeSlowCase(UChar* buffer) const
             if (currentFiberAsRope->isSubstring()) {
                 ASSERT(!currentFiberAsRope->substringBase()->isRope());
                 StringImpl* string = static_cast<StringImpl*>(
-                    currentFiberAsRope->substringBase()->m_value.impl());
+                    currentFiberAsRope->substringBase()->valueInternal().impl());
                 unsigned offset = currentFiberAsRope->substringOffset();
                 unsigned length = currentFiberAsRope->length();
                 position -= length;
@@ -364,11 +405,11 @@ void JSRopeString::resolveRopeSlowCase(UChar* buffer) const
                 continue;
             }
             for (size_t i = 0; i < s_maxInternalRopeLength && currentFiberAsRope->fiber(i); ++i)
-                workQueue.append(currentFiberAsRope->fiber(i).get());
+                workQueue.append(currentFiberAsRope->fiber(i));
             continue;
         }
 
-        StringImpl* string = static_cast<StringImpl*>(currentFiber->m_value.impl());
+        StringImpl* string = static_cast<StringImpl*>(currentFiber->valueInternal().impl());
         unsigned length = string->length();
         position -= length;
         if (string->is8Bit())
@@ -380,16 +421,14 @@ void JSRopeString::resolveRopeSlowCase(UChar* buffer) const
     ASSERT(buffer == position);
 }
 
-void JSRopeString::outOfMemory(ExecState* exec) const
+void JSRopeString::outOfMemory(ExecState* nullOrExecForOOM) const
 {
-    VM& vm = exec->vm();
-    auto scope = DECLARE_THROW_SCOPE(vm);
-
-    clearFibers();
     ASSERT(isRope());
-    ASSERT(m_value.isNull());
-    if (exec)
-        throwOutOfMemoryError(exec, scope);
+    if (nullOrExecForOOM) {
+        VM& vm = nullOrExecForOOM->vm();
+        auto scope = DECLARE_THROW_SCOPE(vm);
+        throwOutOfMemoryError(nullOrExecForOOM, scope);
+    }
 }
 
 JSValue JSString::toPrimitive(ExecState*, PreferredPrimitiveType) const
@@ -444,7 +483,7 @@ bool JSString::getStringPropertyDescriptor(ExecState* exec, PropertyName propert
         return true;
     }
     
-    std::optional<uint32_t> index = parseIndex(propertyName);
+    Optional<uint32_t> index = parseIndex(propertyName);
     if (index && index.value() < length()) {
         descriptor.setDescriptor(getIndex(exec, index.value()), PropertyAttribute::DontDelete | PropertyAttribute::ReadOnly);
         return true;