Unreviewed, rolling in r243948 with test fix
authorysuzuki@apple.com <ysuzuki@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 8 Apr 2019 20:43:18 +0000 (20:43 +0000)
committerysuzuki@apple.com <ysuzuki@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 8 Apr 2019 20:43:18 +0000 (20:43 +0000)
https://bugs.webkit.org/show_bug.cgi?id=196486

JSTests:

* stress/arrow-function-and-use-strict-directive.js: Added.
* stress/arrow-function-syntax.js: Added.
(checkSyntax):
(checkSyntaxError):

Source/JavaScriptCore:

* parser/ASTBuilder.h:
(JSC::ASTBuilder::createString):
* parser/Lexer.cpp:
(JSC::Lexer<T>::parseMultilineComment):
(JSC::Lexer<T>::lexWithoutClearingLineTerminator):
(JSC::Lexer<T>::lex): Deleted.
* parser/Lexer.h:
(JSC::Lexer::hasLineTerminatorBeforeToken const):
(JSC::Lexer::setHasLineTerminatorBeforeToken):
(JSC::Lexer<T>::lex):
(JSC::Lexer::prevTerminator const): Deleted.
(JSC::Lexer::setTerminator): Deleted.
* parser/Parser.cpp:
(JSC::Parser<LexerType>::allowAutomaticSemicolon):
(JSC::Parser<LexerType>::parseSingleFunction):
(JSC::Parser<LexerType>::parseStatementListItem):
(JSC::Parser<LexerType>::maybeParseAsyncFunctionDeclarationStatement):
(JSC::Parser<LexerType>::parseFunctionInfo):
(JSC::Parser<LexerType>::parseClass):
(JSC::Parser<LexerType>::parseExportDeclaration):
(JSC::Parser<LexerType>::parseAssignmentExpression):
(JSC::Parser<LexerType>::parseYieldExpression):
(JSC::Parser<LexerType>::parseProperty):
(JSC::Parser<LexerType>::parsePrimaryExpression):
(JSC::Parser<LexerType>::parseMemberExpression):
* parser/Parser.h:
(JSC::Parser::nextWithoutClearingLineTerminator):
(JSC::Parser::lexCurrentTokenAgainUnderCurrentContext):
(JSC::Parser::internalSaveLexerState):
(JSC::Parser::restoreLexerState):

LayoutTests:

The test relied on the wrong EOF token's offset. This patch also fixes the test.

* inspector/runtime/parse-expected.txt:
* inspector/runtime/parse.html:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@244038 268f45cc-cd09-0410-ab3c-d52691b4dbfc

12 files changed:
JSTests/ChangeLog
JSTests/stress/arrow-function-and-use-strict-directive.js [new file with mode: 0644]
JSTests/stress/arrow-function-syntax.js [new file with mode: 0644]
LayoutTests/ChangeLog
LayoutTests/inspector/runtime/parse-expected.txt
LayoutTests/inspector/runtime/parse.html
Source/JavaScriptCore/ChangeLog
Source/JavaScriptCore/parser/ASTBuilder.h
Source/JavaScriptCore/parser/Lexer.cpp
Source/JavaScriptCore/parser/Lexer.h
Source/JavaScriptCore/parser/Parser.cpp
Source/JavaScriptCore/parser/Parser.h

index 29a741a..5f0cfee 100644 (file)
@@ -1,3 +1,13 @@
+2019-04-08  Yusuke Suzuki  <ysuzuki@apple.com>
+
+        Unreviewed, rolling in r243948 with test fix
+        https://bugs.webkit.org/show_bug.cgi?id=196486
+
+        * stress/arrow-function-and-use-strict-directive.js: Added.
+        * stress/arrow-function-syntax.js: Added.
+        (checkSyntax):
+        (checkSyntaxError):
+
 2019-04-08  Ryan Haddad  <ryanhaddad@apple.com>
 
         Unreviewed, rolling out r243948.
diff --git a/JSTests/stress/arrow-function-and-use-strict-directive.js b/JSTests/stress/arrow-function-and-use-strict-directive.js
new file mode 100644 (file)
index 0000000..2e26516
--- /dev/null
@@ -0,0 +1,18 @@
+// This test should not crash.
+
+dispatch => accessible.children()
+"use strict";
+
+dispatch2 => accessible.children()
+"use strict"
+
+var protected = 42;
+
+dispatch3 => "use strict"
+protected;
+
+async dispatch4 => hey
+"use strict";
+
+async dispatch4 => "use strict"
+protected;
diff --git a/JSTests/stress/arrow-function-syntax.js b/JSTests/stress/arrow-function-syntax.js
new file mode 100644 (file)
index 0000000..d1b7388
--- /dev/null
@@ -0,0 +1,27 @@
+function checkSyntax(src) {
+    try {
+        eval(src);
+    } catch (error) {
+        if (error instanceof SyntaxError)
+            throw new Error("Syntax Error: " + String(error) + "\n script: `" + src + "`");
+    }
+}
+
+function checkSyntaxError(src, message) {
+    var bError = false;
+    try {
+        eval(src);
+    } catch (error) {
+        bError = error instanceof SyntaxError && (String(error) === message || typeof message === 'undefined');
+    }
+    if (!bError) {
+        throw new Error("Expected syntax Error: " + message + "\n in script: `" + src + "`");
+    }
+}
+
+checkSyntax(`()=>42`);
+checkSyntax(`()=>42
+`);
+checkSyntax(`()=>42//Hello`);
+checkSyntax(`()=>42//Hello
+`);
index 773d64a..da85423 100644 (file)
@@ -1,3 +1,13 @@
+2019-04-08  Yusuke Suzuki  <ysuzuki@apple.com>
+
+        Unreviewed, rolling in r243948 with test fix
+        https://bugs.webkit.org/show_bug.cgi?id=196486
+
+        The test relied on the wrong EOF token's offset. This patch also fixes the test.
+
+        * inspector/runtime/parse-expected.txt:
+        * inspector/runtime/parse.html:
+
 2019-04-08  Antti Koivisto  <antti@apple.com>
 
         Update touch-action region on property changes
index 0c57d81..04dfed3 100644 (file)
@@ -75,141 +75,141 @@ Range: {"startOffset":4,"endOffset":9}
 -- Running test case: SyntaxErrorType.Recoverable
 PASS: Should be SyntaxErrorType Recoverable.
 Source: (
-        ^
+         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":1}
+Error Range: {"startOffset":1,"endOffset":1}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: {
-        ^
+         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":1}
+Error Range: {"startOffset":1,"endOffset":1}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: !
-        ^
+         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":1}
+Error Range: {"startOffset":1,"endOffset":1}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: -
-        ^
+         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":1}
+Error Range: {"startOffset":1,"endOffset":1}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: +
-        ^
+         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":1}
+Error Range: {"startOffset":1,"endOffset":1}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: ~
-        ^
+         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":1}
+Error Range: {"startOffset":1,"endOffset":1}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: 1,
-         ^
+          ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":1,"endOffset":2}
+Error Range: {"startOffset":2,"endOffset":2}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var
-        ^~~
+           ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":3}
+Error Range: {"startOffset":3,"endOffset":3}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: 'use strict'; let
-                      ^~~
+                         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":14,"endOffset":17}
+Error Range: {"startOffset":17,"endOffset":17}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: const
-        ^~~~~
+             ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":5}
+Error Range: {"startOffset":5,"endOffset":5}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var {x
-             ^
+              ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":5,"endOffset":6}
+Error Range: {"startOffset":6,"endOffset":6}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var [x
-             ^
+              ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":5,"endOffset":6}
+Error Range: {"startOffset":6,"endOffset":6}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x,
-             ^
+              ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":5,"endOffset":6}
+Error Range: {"startOffset":6,"endOffset":6}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: if(1)
-            ^
+             ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":4,"endOffset":5}
+Error Range: {"startOffset":5,"endOffset":5}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: if(1){
-             ^
+              ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":5,"endOffset":6}
+Error Range: {"startOffset":6,"endOffset":6}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: if(1){}else
-               ^~~~
+                   ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":7,"endOffset":11}
+Error Range: {"startOffset":11,"endOffset":11}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: if(1){}else{
-                   ^
+                    ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":11,"endOffset":12}
+Error Range: {"startOffset":12,"endOffset":12}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: while(1)
-               ^
+                ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":7,"endOffset":8}
+Error Range: {"startOffset":8,"endOffset":8}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: while(1){
-                ^
+                 ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":8,"endOffset":9}
+Error Range: {"startOffset":9,"endOffset":9}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: for(;;)
-              ^
+               ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":6,"endOffset":7}
+Error Range: {"startOffset":7,"endOffset":7}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: for(;;){
-               ^
+                ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":7,"endOffset":8}
+Error Range: {"startOffset":8,"endOffset":8}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: ()=>
-          ^~
+            ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":2,"endOffset":4}
+Error Range: {"startOffset":4,"endOffset":4}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: ()=>{
-            ^
+             ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":4,"endOffset":5}
+Error Range: {"startOffset":5,"endOffset":5}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: /*
@@ -231,201 +231,201 @@ Error Range: {"startOffset":4,"endOffset":5}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: `${
-        ^~~
+           ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":3}
+Error Range: {"startOffset":3,"endOffset":3}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: function
-        ^~~~~~~~
+                ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":8}
+Error Range: {"startOffset":8,"endOffset":8}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: function foo
-                 ^~~
+                    ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":9,"endOffset":12}
+Error Range: {"startOffset":12,"endOffset":12}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: function foo(
-                    ^
+                     ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":12,"endOffset":13}
+Error Range: {"startOffset":13,"endOffset":13}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: function foo(){
-                      ^
+                       ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":14,"endOffset":15}
+Error Range: {"startOffset":15,"endOffset":15}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: (function() {
-                    ^
+                     ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":12,"endOffset":13}
+Error Range: {"startOffset":13,"endOffset":13}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: (function() {}
-                     ^
+                      ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":13,"endOffset":14}
+Error Range: {"startOffset":14,"endOffset":14}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: (function() {})(
-                       ^
+                        ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":15,"endOffset":16}
+Error Range: {"startOffset":16,"endOffset":16}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: switch(x)
-                ^
+                 ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":8,"endOffset":9}
+Error Range: {"startOffset":9,"endOffset":9}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: switch(x){
-                 ^
+                  ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":9,"endOffset":10}
+Error Range: {"startOffset":10,"endOffset":10}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: switch(x){case
-                  ^~~~
+                      ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":10,"endOffset":14}
+Error Range: {"startOffset":14,"endOffset":14}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: switch(x){case 1
-                       ^
+                        ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":15,"endOffset":16}
+Error Range: {"startOffset":16,"endOffset":16}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: switch(x){case 1:
-                        ^
+                         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":16,"endOffset":17}
+Error Range: {"startOffset":17,"endOffset":17}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: class
-        ^~~~~
+             ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":0,"endOffset":5}
+Error Range: {"startOffset":5,"endOffset":5}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: class Klass
-              ^~~~~
+                   ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":6,"endOffset":11}
+Error Range: {"startOffset":11,"endOffset":11}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: class Klass {
-                    ^
+                     ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":12,"endOffset":13}
+Error Range: {"startOffset":13,"endOffset":13}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {
-                ^
+                 ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":8,"endOffset":9}
+Error Range: {"startOffset":9,"endOffset":9}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop
-                 ^~~~
+                     ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":9,"endOffset":13}
+Error Range: {"startOffset":13,"endOffset":13}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:
-                     ^
+                      ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":13,"endOffset":14}
+Error Range: {"startOffset":14,"endOffset":14}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123
-                      ^~~
+                         ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":14,"endOffset":17}
+Error Range: {"startOffset":17,"endOffset":17}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123 
-                      ^~~
+                          ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":14,"endOffset":17}
+Error Range: {"startOffset":18,"endOffset":18}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123,
-                         ^
+                          ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":17,"endOffset":18}
+Error Range: {"startOffset":18,"endOffset":18}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123, 
-                         ^
+                           ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":17,"endOffset":18}
+Error Range: {"startOffset":19,"endOffset":19}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123, 'prop2'
-                           ^~~~~~~
+                                  ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":19,"endOffset":26}
+Error Range: {"startOffset":26,"endOffset":26}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123, [12
-                            ^~
+                              ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":20,"endOffset":22}
+Error Range: {"startOffset":22,"endOffset":22}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123, [12]
-                              ^
+                               ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":22,"endOffset":23}
+Error Range: {"startOffset":23,"endOffset":23}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123, [12]:
-                               ^
+                                ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":23,"endOffset":24}
+Error Range: {"startOffset":24,"endOffset":24}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = {prop:123, [12]:567
-                                ^~~
+                                   ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":24,"endOffset":27}
+Error Range: {"startOffset":27,"endOffset":27}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = [
-                ^
+                 ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":8,"endOffset":9}
+Error Range: {"startOffset":9,"endOffset":9}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = [1
-                 ^
+                  ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":9,"endOffset":10}
+Error Range: {"startOffset":10,"endOffset":10}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = [1,
-                  ^
+                   ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":10,"endOffset":11}
+Error Range: {"startOffset":11,"endOffset":11}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = [1,[]
-                    ^
+                     ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":12,"endOffset":13}
+Error Range: {"startOffset":13,"endOffset":13}
 
 PASS: Should be SyntaxErrorType Recoverable.
 Source: var x = [1,{}
-                    ^
+                     ^
 Error Message: Unexpected end of script
-Error Range: {"startOffset":12,"endOffset":13}
+Error Range: {"startOffset":13,"endOffset":13}
 
 
 -- Running test case: SyntaxErrorType.Irrecoverable
index 150c56d..98d8ab1 100644 (file)
@@ -9,7 +9,7 @@ function test()
         var str = " ".repeat("Source: ".length);
         str += " ".repeat(range.startOffset);
         str += "^";
-        str += "~".repeat(range.endOffset - range.startOffset - 1);
+        str += "~".repeat(Math.max(range.endOffset - range.startOffset - 1, 0));
         InspectorTest.log(str);
     }
 
index 78491ec..9c64d7d 100644 (file)
@@ -1,3 +1,39 @@
+2019-04-08  Yusuke Suzuki  <ysuzuki@apple.com>
+
+        Unreviewed, rolling in r243948 with test fix
+        https://bugs.webkit.org/show_bug.cgi?id=196486
+
+        * parser/ASTBuilder.h:
+        (JSC::ASTBuilder::createString):
+        * parser/Lexer.cpp:
+        (JSC::Lexer<T>::parseMultilineComment):
+        (JSC::Lexer<T>::lexWithoutClearingLineTerminator):
+        (JSC::Lexer<T>::lex): Deleted.
+        * parser/Lexer.h:
+        (JSC::Lexer::hasLineTerminatorBeforeToken const):
+        (JSC::Lexer::setHasLineTerminatorBeforeToken):
+        (JSC::Lexer<T>::lex):
+        (JSC::Lexer::prevTerminator const): Deleted.
+        (JSC::Lexer::setTerminator): Deleted.
+        * parser/Parser.cpp:
+        (JSC::Parser<LexerType>::allowAutomaticSemicolon):
+        (JSC::Parser<LexerType>::parseSingleFunction):
+        (JSC::Parser<LexerType>::parseStatementListItem):
+        (JSC::Parser<LexerType>::maybeParseAsyncFunctionDeclarationStatement):
+        (JSC::Parser<LexerType>::parseFunctionInfo):
+        (JSC::Parser<LexerType>::parseClass):
+        (JSC::Parser<LexerType>::parseExportDeclaration):
+        (JSC::Parser<LexerType>::parseAssignmentExpression):
+        (JSC::Parser<LexerType>::parseYieldExpression):
+        (JSC::Parser<LexerType>::parseProperty):
+        (JSC::Parser<LexerType>::parsePrimaryExpression):
+        (JSC::Parser<LexerType>::parseMemberExpression):
+        * parser/Parser.h:
+        (JSC::Parser::nextWithoutClearingLineTerminator):
+        (JSC::Parser::lexCurrentTokenAgainUnderCurrentContext):
+        (JSC::Parser::internalSaveLexerState):
+        (JSC::Parser::restoreLexerState):
+
 2019-04-08  Ryan Haddad  <ryanhaddad@apple.com>
 
         Unreviewed, rolling out r243948.
index 622d291..5f98503 100644 (file)
@@ -241,6 +241,7 @@ public:
 
     ExpressionNode* createString(const JSTokenLocation& location, const Identifier* string)
     {
+        ASSERT(string);
         incConstants();
         return new (m_parserArena) StringNode(location, *string);
     }
index 9dc9de3..c2f24bd 100644 (file)
@@ -1691,7 +1691,7 @@ ALWAYS_INLINE bool Lexer<T>::parseMultilineComment()
 
         if (isLineTerminator(m_current)) {
             shiftLineTerminator();
-            m_terminator = true;
+            m_hasLineTerminatorBeforeToken = true;
         } else
             shift();
     }
@@ -1770,7 +1770,7 @@ void Lexer<T>::fillTokenInfo(JSToken* tokenRecord, JSTokenType token, int lineNu
 }
 
 template <typename T>
-JSTokenType Lexer<T>::lex(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
+JSTokenType Lexer<T>::lexWithoutClearingLineTerminator(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
 {
     JSTokenData* tokenData = &tokenRecord->m_data;
     JSTokenLocation* tokenLocation = &tokenRecord->m_location;
@@ -1781,18 +1781,19 @@ JSTokenType Lexer<T>::lex(JSToken* tokenRecord, unsigned lexerFlags, bool strict
     ASSERT(m_buffer16.isEmpty());
 
     JSTokenType token = ERRORTOK;
-    m_terminator = false;
 
 start:
     skipWhitespace();
 
-    if (atEnd())
-        return EOFTOK;
-    
     tokenLocation->startOffset = currentOffset();
     ASSERT(currentOffset() >= currentLineStartOffset());
     tokenRecord->m_startPosition = currentPosition();
 
+    if (atEnd()) {
+        token = EOFTOK;
+        goto returnToken;
+    }
+
     CharacterType type;
     if (LIKELY(isLatin1(m_current)))
         type = static_cast<CharacterType>(typesOfLatin1Characters[m_current]);
@@ -1902,7 +1903,7 @@ start:
         shift();
         if (m_current == '+') {
             shift();
-            token = (!m_terminator) ? PLUSPLUS : AUTOPLUSPLUS;
+            token = (!m_hasLineTerminatorBeforeToken) ? PLUSPLUS : AUTOPLUSPLUS;
             break;
         }
         if (m_current == '=') {
@@ -1916,13 +1917,13 @@ start:
         shift();
         if (m_current == '-') {
             shift();
-            if ((m_atLineStart || m_terminator) && m_current == '>') {
+            if ((m_atLineStart || m_hasLineTerminatorBeforeToken) && m_current == '>') {
                 if (m_scriptMode == JSParserScriptMode::Classic) {
                     shift();
                     goto inSingleLineComment;
                 }
             }
-            token = (!m_terminator) ? MINUSMINUS : AUTOMINUSMINUS;
+            token = (!m_hasLineTerminatorBeforeToken) ? MINUSMINUS : AUTOMINUSMINUS;
             break;
         }
         if (m_current == '=') {
@@ -2293,7 +2294,7 @@ start:
         ASSERT(isLineTerminator(m_current));
         shiftLineTerminator();
         m_atLineStart = true;
-        m_terminator = true;
+        m_hasLineTerminatorBeforeToken = true;
         m_lineStart = m_code;
         goto start;
     case CharacterPrivateIdentifierStart:
@@ -2333,13 +2334,16 @@ inSingleLineComment:
         auto endPosition = currentPosition();
 
         while (!isLineTerminator(m_current)) {
-            if (atEnd())
-                return EOFTOK;
+            if (atEnd()) {
+                token = EOFTOK;
+                fillTokenInfo(tokenRecord, token, lineNumber, endOffset, lineStartOffset, endPosition);
+                return token;
+            }
             shift();
         }
         shiftLineTerminator();
         m_atLineStart = true;
-        m_terminator = true;
+        m_hasLineTerminatorBeforeToken = true;
         m_lineStart = m_code;
         if (!lastTokenWasRestrKeyword())
             goto start;
index 700d1a7..3755437 100644 (file)
@@ -65,6 +65,7 @@ public:
     bool isReparsingFunction() const { return m_isReparsingFunction; }
 
     JSTokenType lex(JSToken*, unsigned, bool strictMode);
+    JSTokenType lexWithoutClearingLineTerminator(JSToken*, unsigned, bool strictMode);
     bool nextTokenIsColon();
     int lineNumber() const { return m_lineNumber; }
     ALWAYS_INLINE int currentOffset() const { return offsetFromSourcePtr(m_code); }
@@ -77,7 +78,7 @@ public:
     JSTokenLocation lastTokenLocation() const { return m_lastTokenLocation; }
     void setLastLineNumber(int lastLineNumber) { m_lastLineNumber = lastLineNumber; }
     int lastLineNumber() const { return m_lastLineNumber; }
-    bool prevTerminator() const { return m_terminator; }
+    bool hasLineTerminatorBeforeToken() const { return m_hasLineTerminatorBeforeToken; }
     JSTokenType scanRegExp(JSToken*, UChar patternPrefix = 0);
     enum class RawStringsBuildMode { BuildRawStrings, DontBuildRawStrings };
     JSTokenType scanTemplateString(JSToken*, RawStringsBuildMode);
@@ -110,9 +111,9 @@ public:
     {
         m_lineNumber = line;
     }
-    void setTerminator(bool terminator)
+    void setHasLineTerminatorBeforeToken(bool terminator)
     {
-        m_terminator = terminator;
+        m_hasLineTerminatorBeforeToken = terminator;
     }
 
     JSTokenType lexExpectIdentifier(JSToken*, unsigned, bool strictMode);
@@ -202,7 +203,7 @@ private:
     Vector<LChar> m_buffer8;
     Vector<UChar> m_buffer16;
     Vector<UChar> m_bufferForRawTemplateString16;
-    bool m_terminator;
+    bool m_hasLineTerminatorBeforeToken;
     int m_lastToken;
 
     const SourceCode* m_source;
@@ -403,4 +404,11 @@ slowCase:
     return lex(tokenRecord, lexerFlags, strictMode);
 }
 
+template <typename T>
+ALWAYS_INLINE JSTokenType Lexer<T>::lex(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
+{
+    m_hasLineTerminatorBeforeToken = false;
+    return lexWithoutClearingLineTerminator(tokenRecord, lexerFlags, strictMode);
+}
+
 } // namespace JSC
index ba01bfb..dce4551 100644 (file)
@@ -345,7 +345,7 @@ bool Parser<LexerType>::isArrowFunctionParameters()
 template <typename LexerType>
 bool Parser<LexerType>::allowAutomaticSemicolon()
 {
-    return match(CLOSEBRACE) || match(EOFTOK) || m_lexer->prevTerminator();
+    return match(CLOSEBRACE) || match(EOFTOK) || m_lexer->hasLineTerminatorBeforeToken();
 }
 
 template <typename LexerType>
@@ -625,7 +625,7 @@ template <class TreeBuilder> TreeSourceElements Parser<LexerType>::parseSingleFu
     case IDENT:
         if (*m_token.m_data.ident == m_vm->propertyNames->async && !m_token.m_data.escaped) {
             next();
-            failIfFalse(match(FUNCTION) && !m_lexer->prevTerminator(), "Cannot parse the async function");
+            failIfFalse(match(FUNCTION) && !m_lexer->hasLineTerminatorBeforeToken(), "Cannot parse the async function");
             statement = parseAsyncFunctionDeclaration(context, ExportType::NotExported, DeclarationDefaultContext::Standard, functionConstructorParametersEndPosition);
             break;
         }
@@ -696,7 +696,7 @@ template <class TreeBuilder> TreeStatement Parser<LexerType>::parseStatementList
             // but could be mistakenly parsed as an AsyncFunctionExpression.
             SavePoint savePoint = createSavePoint();
             next();
-            if (UNLIKELY(match(FUNCTION) && !m_lexer->prevTerminator())) {
+            if (UNLIKELY(match(FUNCTION) && !m_lexer->hasLineTerminatorBeforeToken())) {
                 result = parseAsyncFunctionDeclaration(context);
                 break;
             }
@@ -2026,7 +2026,7 @@ template <class TreeBuilder> bool Parser<LexerType>::maybeParseAsyncFunctionDecl
     ASSERT(matchContextualKeyword(m_vm->propertyNames->async));
     SavePoint savePoint = createSavePoint();
     next();
-    if (match(FUNCTION) && !m_lexer->prevTerminator()) {
+    if (match(FUNCTION) && !m_lexer->hasLineTerminatorBeforeToken()) {
         const bool isAsync = true;
         result = parseFunctionDeclarationStatement(context, isAsync, parentAllowsFunctionDeclarationAsStatement);
         return true;
@@ -2421,7 +2421,7 @@ template <class TreeBuilder> bool Parser<LexerType>::parseFunctionInfo(TreeBuild
 
         matchOrFail(ARROWFUNCTION, "Expected a '=>' after arrow function parameter declaration");
 
-        if (m_lexer->prevTerminator())
+        if (m_lexer->hasLineTerminatorBeforeToken())
             failDueToUnexpectedToken();
 
         ASSERT(constructorKind == ConstructorKind::None);
@@ -2612,12 +2612,23 @@ template <class TreeBuilder> bool Parser<LexerType>::parseFunctionInfo(TreeBuild
         functionScope->fillParametersForSourceProviderCache(parameters, nonLocalCapturesFromParameterExpressions);
         newInfo = SourceProviderCacheItem::create(parameters);
     }
+
+    bool functionScopeWasStrictMode = functionScope->strictMode();
     
     popScope(functionScope, TreeBuilder::NeedsFreeVariableInfo);
     
     if (functionBodyType != ArrowFunctionBodyExpression) {
         matchOrFail(CLOSEBRACE, "Expected a closing '}' after a ", stringForFunctionMode(mode), " body");
         next();
+    } else {
+        // We need to lex the last token again because the last token is lexed under the different context because of the following possibilities.
+        // 1. which may have different strict mode.
+        // 2. which may not build strings for tokens.
+        // But (1) is not possible because we do not recognize the string literal in ArrowFunctionBodyExpression as directive and this is correct in terms of the spec (`value => "use strict"`).
+        // So we only check TreeBuilder's type here.
+        ASSERT_UNUSED(functionScopeWasStrictMode, functionScopeWasStrictMode == currentScope()->strictMode());
+        if (!std::is_same<TreeBuilder, SyntaxChecker>::value)
+            lexCurrentTokenAgainUnderCurrentContext();
     }
 
     if (newInfo)
@@ -2876,7 +2887,7 @@ parseMethod:
                 if (!isGeneratorMethodParseMode(parseMode) && !isAsyncMethodParseMode(parseMode)) {
                     ident = m_token.m_data.ident;
                     next();
-                    if (match(OPENPAREN) || match(COLON) || match(EQUAL) || m_lexer->prevTerminator())
+                    if (match(OPENPAREN) || match(COLON) || match(EQUAL) || m_lexer->hasLineTerminatorBeforeToken())
                         break;
                     if (UNLIKELY(consume(TIMES)))
                         parseMode = SourceParseMode::AsyncGeneratorWrapperMethodMode;
@@ -3396,7 +3407,7 @@ template <class TreeBuilder> TreeStatement Parser<LexerType>::parseExportDeclara
         } else if (matchContextualKeyword(m_vm->propertyNames->async)) {
             SavePoint savePoint = createSavePoint();
             next();
-            if (match(FUNCTION) && !m_lexer->prevTerminator()) {
+            if (match(FUNCTION) && !m_lexer->hasLineTerminatorBeforeToken()) {
                 next();
                 if (match(IDENT))
                     localName = m_token.m_data.ident;
@@ -3541,7 +3552,7 @@ template <class TreeBuilder> TreeStatement Parser<LexerType>::parseExportDeclara
         case IDENT:
             if (*m_token.m_data.ident == m_vm->propertyNames->async && !m_token.m_data.escaped) {
                 next();
-                semanticFailIfFalse(match(FUNCTION) && !m_lexer->prevTerminator(), "Expected 'function' keyword following 'async' keyword with no preceding line terminator");
+                semanticFailIfFalse(match(FUNCTION) && !m_lexer->hasLineTerminatorBeforeToken(), "Expected 'function' keyword following 'async' keyword with no preceding line terminator");
                 DepthManager statementDepth(&m_statementDepth);
                 m_statementDepth = 1;
                 result = parseAsyncFunctionDeclaration(context, ExportType::Exported);
@@ -3659,7 +3670,7 @@ template <typename TreeBuilder> TreeExpression Parser<LexerType>::parseAssignmen
             if (UNLIKELY(classifier.indicatesPossibleAsyncArrowFunction())) {
                 if (matchContextualKeyword(m_vm->propertyNames->async)) {
                     next();
-                    isAsyncArrow = !m_lexer->prevTerminator();
+                    isAsyncArrow = !m_lexer->hasLineTerminatorBeforeToken();
                 }
             }
             if (isArrowFunctionParameters()) {
@@ -3776,7 +3787,7 @@ template <class TreeBuilder> TreeExpression Parser<LexerType>::parseYieldExpress
     ASSERT(match(YIELD));
     SavePoint savePoint = createSavePoint();
     next();
-    if (m_lexer->prevTerminator())
+    if (m_lexer->hasLineTerminatorBeforeToken())
         return context.createYield(location);
 
     bool delegate = consume(TIMES);
@@ -3936,7 +3947,7 @@ parseProperty:
                     goto namedProperty;
                 }
 
-                failIfTrue(m_lexer->prevTerminator(), "Expected a property name following keyword 'async'");
+                failIfTrue(m_lexer->hasLineTerminatorBeforeToken(), "Expected a property name following keyword 'async'");
                 if (UNLIKELY(consume(TIMES)))
                     parseMode = SourceParseMode::AsyncGeneratorWrapperMethodMode;
                 else
@@ -4485,7 +4496,7 @@ template <class TreeBuilder> TreeExpression Parser<LexerType>::parsePrimaryExpre
             const Identifier* ident = m_token.m_data.ident;
             JSTokenLocation location(tokenLocation());
             next();
-            if (match(FUNCTION) && !m_lexer->prevTerminator())
+            if (match(FUNCTION) && !m_lexer->hasLineTerminatorBeforeToken())
                 return parseAsyncFunctionExpression(context);
 
             // Avoid using variable if it is an arrow function parameter
@@ -4751,7 +4762,7 @@ template <class TreeBuilder> TreeExpression Parser<LexerType>::parseMemberExpres
 
         base = parsePrimaryExpression(context);
         failIfFalse(base, "Cannot parse base expression");
-        if (UNLIKELY(isAsync && context.isResolve(base) && !m_lexer->prevTerminator())) {
+        if (UNLIKELY(isAsync && context.isResolve(base) && !m_lexer->hasLineTerminatorBeforeToken())) {
             if (matchSpecIdentifier()) {
                 // AsyncArrowFunction
                 forceClassifyExpressionError(ErrorIndicatesAsyncArrowFunction);
index d1f9434..8158993 100644 (file)
@@ -1363,6 +1363,16 @@ private:
         m_token.m_type = m_lexer->lex(&m_token, lexerFlags, strictMode());
     }
 
+    ALWAYS_INLINE void nextWithoutClearingLineTerminator(unsigned lexerFlags = 0)
+    {
+        int lastLine = m_token.m_location.line;
+        int lastTokenEnd = m_token.m_location.endOffset;
+        int lastTokenLineStart = m_token.m_location.lineStartOffset;
+        m_lastTokenEndPosition = JSTextPosition(lastLine, lastTokenEnd, lastTokenLineStart);
+        m_lexer->setLastLineNumber(lastLine);
+        m_token.m_type = m_lexer->lexWithoutClearingLineTerminator(&m_token, lexerFlags, strictMode());
+    }
+
     ALWAYS_INLINE void nextExpectIdentifier(unsigned lexerFlags = 0)
     {
         int lastLine = m_token.m_location.line;
@@ -1373,6 +1383,12 @@ private:
         m_token.m_type = m_lexer->lexExpectIdentifier(&m_token, lexerFlags, strictMode());
     }
 
+    ALWAYS_INLINE void lexCurrentTokenAgainUnderCurrentContext()
+    {
+        auto savePoint = createSavePoint();
+        restoreSavePoint(savePoint);
+    }
+
     ALWAYS_INLINE bool nextTokenIsColon()
     {
         return m_lexer->nextTokenIsColon();
@@ -1762,6 +1778,7 @@ private:
         unsigned oldLineStartOffset;
         unsigned oldLastLineNumber;
         unsigned oldLineNumber;
+        bool hasLineTerminatorBeforeToken;
     };
 
     // If you're using this directly, you probably should be using
@@ -1775,6 +1792,7 @@ private:
         result.oldLineStartOffset = m_token.m_location.lineStartOffset;
         result.oldLastLineNumber = m_lexer->lastLineNumber();
         result.oldLineNumber = m_lexer->lineNumber();
+        result.hasLineTerminatorBeforeToken = m_lexer->hasLineTerminatorBeforeToken();
         ASSERT(static_cast<unsigned>(result.startOffset) >= result.oldLineStartOffset);
         return result;
     }
@@ -1784,7 +1802,8 @@ private:
         // setOffset clears lexer errors.
         m_lexer->setOffset(lexerState.startOffset, lexerState.oldLineStartOffset);
         m_lexer->setLineNumber(lexerState.oldLineNumber);
-        next();
+        m_lexer->setHasLineTerminatorBeforeToken(lexerState.hasLineTerminatorBeforeToken);
+        nextWithoutClearingLineTerminator();
         m_lexer->setLastLineNumber(lexerState.oldLastLineNumber);
     }