1 // -*- c-basic-offset: 2 -*-
3 * Copyright (C) 1999-2000 Harri Porten (porten@kde.org)
4 * Copyright (C) 2006, 2007 Apple Inc. All Rights Reserved.
5 * Copyright (C) 2007 Cameron Zwarich (cwzwarich@uwaterloo.ca)
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
17 * You should have received a copy of the GNU Library General Public License
18 * along with this library; see the file COPYING.LIB. If not, write to
19 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
32 #include <wtf/Assertions.h>
33 #include <wtf/unicode/Unicode.h>
36 using namespace Unicode;
38 // we can't specify the namespace in yacc's C output, so do it here
46 #include "lexer.lut.h"
48 extern YYLTYPE kjsyylloc; // global bison variable holding token info
50 // a bridge for yacc from the C world to C++
58 static bool isDecimalDigit(int);
60 static const size_t initialReadBufferCapacity = 32;
61 static const size_t initialStringTableCapacity = 64;
65 ASSERT(JSLock::currentThreadIsHoldingLock());
67 // FIXME: We'd like to avoid calling new here, but we don't currently
68 // support tearing down the Lexer at app quit time, since that would involve
69 // tearing down its UString data members without holding the JSLock.
70 static Lexer* staticLexer = new Lexer;
77 , eatNextIdentifier(false)
91 m_buffer8.reserveCapacity(initialReadBufferCapacity);
92 m_buffer16.reserveCapacity(initialReadBufferCapacity);
93 m_strings.reserveCapacity(initialStringTableCapacity);
94 m_identifiers.reserveCapacity(initialStringTableCapacity);
97 void Lexer::setCode(int startingLineNumber, const KJS::UChar *c, unsigned int len)
99 yylineno = 1 + startingLineNumber;
100 restrKeyword = false;
102 eatNextIdentifier = false;
111 #ifndef KJS_PURE_ECMA
115 // read first characters
116 current = (length > 0) ? code[0].uc : -1;
117 next1 = (length > 1) ? code[1].uc : -1;
118 next2 = (length > 2) ? code[2].uc : -1;
119 next3 = (length > 3) ? code[3].uc : -1;
122 void Lexer::shift(unsigned int p)
124 // Here would be a good place to strip Cf characters, but that has caused compatibility problems:
125 // <http://bugs.webkit.org/show_bug.cgi?id=10183>.
131 next3 = (pos + 3 < length) ? code[pos + 3].uc : -1;
135 // called on each new line
136 void Lexer::nextLine()
139 #ifndef KJS_PURE_ECMA
144 void Lexer::setDone(State s)
154 unsigned short stringType = 0; // either single or double quotes
162 // did we push a token on the stack previously ?
163 // (after an automatic semicolon insertion)
164 if (stackToken >= 0) {
171 if (skipLF && current != '\n') // found \r but not \n afterwards
173 if (skipCR && current != '\r') // found \n but not \r afterwards
175 if (skipLF || skipCR) // found \r\n or \n\r -> eat the second one
183 if (isWhiteSpace()) {
185 } else if (current == '/' && next1 == '/') {
187 state = InSingleLineComment;
188 } else if (current == '/' && next1 == '*') {
190 state = InMultiLineComment;
191 } else if (current == -1) {
192 if (!terminator && !delimited) {
193 // automatic semicolon insertion if program incomplete
199 } else if (isLineTerminator()) {
206 } else if (current == '"' || current == '\'') {
208 stringType = static_cast<unsigned short>(current);
209 } else if (isIdentStart(current)) {
211 state = InIdentifierOrKeyword;
212 } else if (current == '\\') {
213 state = InIdentifierUnicodeEscapeStart;
214 } else if (current == '0') {
217 } else if (isDecimalDigit(current)) {
220 } else if (current == '.' && isDecimalDigit(next1)) {
223 #ifndef KJS_PURE_ECMA
224 // <!-- marks the beginning of a line comment (for www usage)
225 } else if (current == '<' && next1 == '!' &&
226 next2 == '-' && next3 == '-') {
228 state = InSingleLineComment;
230 } else if (bol && current == '-' && next1 == '-' && next2 == '>') {
232 state = InSingleLineComment;
235 token = matchPunctuator(current, next1, next2, next3);
239 // cerr << "encountered unknown character" << endl;
245 if (current == stringType) {
248 } else if (isLineTerminator() || current == -1) {
250 } else if (current == '\\') {
251 state = InEscapeSequence;
256 // Escape Sequences inside of strings
257 case InEscapeSequence:
258 if (isOctalDigit(current)) {
259 if (current >= '0' && current <= '3' &&
260 isOctalDigit(next1) && isOctalDigit(next2)) {
261 record16(convertOctal(current, next1, next2));
264 } else if (isOctalDigit(current) && isOctalDigit(next1)) {
265 record16(convertOctal('0', current, next1));
268 } else if (isOctalDigit(current)) {
269 record16(convertOctal('0', '0', current));
274 } else if (current == 'x')
276 else if (current == 'u')
277 state = InUnicodeEscape;
278 else if (isLineTerminator()) {
282 record16(singleEscape(static_cast<unsigned short>(current)));
287 if (isHexDigit(current) && isHexDigit(next1)) {
289 record16(convertHex(current, next1));
291 } else if (current == stringType) {
301 case InUnicodeEscape:
302 if (isHexDigit(current) && isHexDigit(next1) && isHexDigit(next2) && isHexDigit(next3)) {
303 record16(convertUnicode(current, next1, next2, next3));
306 } else if (current == stringType) {
314 case InSingleLineComment:
315 if (isLineTerminator()) {
323 } else if (current == -1) {
327 case InMultiLineComment:
330 } else if (isLineTerminator()) {
332 } else if (current == '*' && next1 == '/') {
337 case InIdentifierOrKeyword:
339 if (isIdentPart(current))
341 else if (current == '\\')
342 state = InIdentifierUnicodeEscapeStart;
344 setDone(state == InIdentifierOrKeyword ? IdentifierOrKeyword : Identifier);
347 if (current == 'x' || current == 'X') {
350 } else if (current == '.') {
353 } else if (current == 'e' || current == 'E') {
355 state = InExponentIndicator;
356 } else if (isOctalDigit(current)) {
359 } else if (isDecimalDigit(current)) {
367 if (isHexDigit(current)) {
374 if (isOctalDigit(current)) {
377 else if (isDecimalDigit(current)) {
384 if (isDecimalDigit(current)) {
386 } else if (current == '.') {
389 } else if (current == 'e' || current == 'E') {
391 state = InExponentIndicator;
396 if (isDecimalDigit(current)) {
398 } else if (current == 'e' || current == 'E') {
400 state = InExponentIndicator;
404 case InExponentIndicator:
405 if (current == '+' || current == '-') {
407 } else if (isDecimalDigit(current)) {
414 if (isDecimalDigit(current)) {
419 case InIdentifierUnicodeEscapeStart:
421 state = InIdentifierUnicodeEscape;
425 case InIdentifierUnicodeEscape:
426 if (isHexDigit(current) && isHexDigit(next1) && isHexDigit(next2) && isHexDigit(next3)) {
427 record16(convertUnicode(current, next1, next2, next3));
429 state = InIdentifier;
435 ASSERT(!"Unhandled state in switch statement");
438 // move on to the next character
441 #ifndef KJS_PURE_ECMA
442 if (state != Start && state != InSingleLineComment)
447 // no identifiers allowed directly after numeric literal, e.g. "3in" is bad
448 if ((state == Number || state == Octal || state == Hex) && isIdentStart(current))
452 m_buffer8.append('\0');
455 fprintf(stderr, "line: %d ", lineNo());
456 fprintf(stderr, "yytext (%x): ", m_buffer8[0]);
457 fprintf(stderr, "%s ", buffer8.data());
461 if (state == Number) {
462 dval = strtod(m_buffer8.data(), 0L);
463 } else if (state == Hex) { // scan hex numbers
464 const char* p = m_buffer8.data() + 2;
465 while (char c = *p++) {
467 dval += convertHex(c);
470 if (dval >= mantissaOverflowLowerBound)
471 dval = parseIntOverflow(m_buffer8.data() + 2, p - (m_buffer8.data() + 3), 16);
474 } else if (state == Octal) { // scan octal number
475 const char* p = m_buffer8.data() + 1;
476 while (char c = *p++) {
481 if (dval >= mantissaOverflowLowerBound)
482 dval = parseIntOverflow(m_buffer8.data() + 1, p - (m_buffer8.data() + 2), 8);
496 printf("(Identifier)/(Keyword)\n");
499 printf("(String)\n");
502 printf("(Number)\n");
509 if (state != Identifier && eatNextIdentifier)
510 eatNextIdentifier = false;
512 restrKeyword = false;
514 kjsyylloc.first_line = yylineno; // ???
515 kjsyylloc.last_line = yylineno;
522 if(token == '}' || token == ';') {
526 case IdentifierOrKeyword:
527 if ((token = Lookup::find(&mainTable, m_buffer16.data(), m_buffer16.size())) < 0) {
529 // Lookup for keyword failed, means this is an identifier
530 // Apply anonymous-function hack below (eat the identifier)
531 if (eatNextIdentifier) {
532 eatNextIdentifier = false;
536 kjsyylval.ident = makeIdentifier(m_buffer16);
541 eatNextIdentifier = false;
542 // Hack for "f = function somename() { ... }", too hard to get into the grammar
543 if (token == FUNCTION && lastToken == '=' )
544 eatNextIdentifier = true;
546 if (token == CONTINUE || token == BREAK ||
547 token == RETURN || token == THROW)
551 kjsyylval.string = makeUString(m_buffer16);
555 kjsyylval.doubleValue = dval;
560 fprintf(stderr, "yylex: ERROR.\n");
565 ASSERT(!"unhandled numeration value in switch");
573 bool Lexer::isWhiteSpace() const
575 return current == '\t' || current == 0x0b || current == 0x0c || isSeparatorSpace(current);
578 bool Lexer::isLineTerminator()
580 bool cr = (current == '\r');
581 bool lf = (current == '\n');
586 return cr || lf || current == 0x2028 || current == 0x2029;
589 bool Lexer::isIdentStart(int c)
591 return (category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other))
592 || c == '$' || c == '_';
595 bool Lexer::isIdentPart(int c)
597 return (category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other
598 | Mark_NonSpacing | Mark_SpacingCombining | Number_DecimalDigit | Punctuation_Connector))
599 || c == '$' || c == '_';
602 static bool isDecimalDigit(int c)
604 return (c >= '0' && c <= '9');
607 bool Lexer::isHexDigit(int c)
609 return (c >= '0' && c <= '9' ||
610 c >= 'a' && c <= 'f' ||
611 c >= 'A' && c <= 'F');
614 bool Lexer::isOctalDigit(int c)
616 return (c >= '0' && c <= '7');
619 int Lexer::matchPunctuator(int c1, int c2, int c3, int c4)
621 if (c1 == '>' && c2 == '>' && c3 == '>' && c4 == '=') {
624 } else if (c1 == '=' && c2 == '=' && c3 == '=') {
627 } else if (c1 == '!' && c2 == '=' && c3 == '=') {
630 } else if (c1 == '>' && c2 == '>' && c3 == '>') {
633 } else if (c1 == '<' && c2 == '<' && c3 == '=') {
636 } else if (c1 == '>' && c2 == '>' && c3 == '=') {
639 } else if (c1 == '<' && c2 == '=') {
642 } else if (c1 == '>' && c2 == '=') {
645 } else if (c1 == '!' && c2 == '=') {
648 } else if (c1 == '+' && c2 == '+') {
654 } else if (c1 == '-' && c2 == '-') {
657 return AUTOMINUSMINUS;
660 } else if (c1 == '=' && c2 == '=') {
663 } else if (c1 == '+' && c2 == '=') {
666 } else if (c1 == '-' && c2 == '=') {
669 } else if (c1 == '*' && c2 == '=') {
672 } else if (c1 == '/' && c2 == '=') {
675 } else if (c1 == '&' && c2 == '=') {
678 } else if (c1 == '^' && c2 == '=') {
681 } else if (c1 == '%' && c2 == '=') {
684 } else if (c1 == '|' && c2 == '=') {
687 } else if (c1 == '<' && c2 == '<') {
690 } else if (c1 == '>' && c2 == '>') {
693 } else if (c1 == '&' && c2 == '&') {
696 } else if (c1 == '|' && c2 == '|') {
727 return static_cast<int>(c1);
733 unsigned short Lexer::singleEscape(unsigned short c)
759 unsigned short Lexer::convertOctal(int c1, int c2, int c3)
761 return static_cast<unsigned short>((c1 - '0') * 64 + (c2 - '0') * 8 + c3 - '0');
764 unsigned char Lexer::convertHex(int c)
766 if (c >= '0' && c <= '9')
767 return static_cast<unsigned char>(c - '0');
768 if (c >= 'a' && c <= 'f')
769 return static_cast<unsigned char>(c - 'a' + 10);
770 return static_cast<unsigned char>(c - 'A' + 10);
773 unsigned char Lexer::convertHex(int c1, int c2)
775 return ((convertHex(c1) << 4) + convertHex(c2));
778 KJS::UChar Lexer::convertUnicode(int c1, int c2, int c3, int c4)
780 // FIXME: This conversion is lossy. See http://bugs.webkit.org/show_bug.cgi?id=4920.
781 return KJS::UChar((convertHex(c1) << 4) + convertHex(c2),
782 (convertHex(c3) << 4) + convertHex(c4));
785 void Lexer::record8(int c)
789 m_buffer8.append(static_cast<char>(c));
792 void Lexer::record16(int c)
795 ASSERT(c <= USHRT_MAX);
796 record16(UChar(static_cast<unsigned short>(c)));
799 void Lexer::record16(KJS::UChar c)
801 m_buffer16.append(c);
804 bool Lexer::scanRegExp()
807 bool lastWasEscape = false;
808 bool inBrackets = false;
811 if (isLineTerminator() || current == -1)
813 else if (current != '/' || lastWasEscape == true || inBrackets == true)
815 // keep track of '[' and ']'
816 if (!lastWasEscape) {
817 if ( current == '[' && !inBrackets )
819 if ( current == ']' && inBrackets )
824 !lastWasEscape && (current == '\\');
825 } else { // end of regexp
826 m_pattern = UString(m_buffer16);
834 while (isIdentPart(current)) {
838 m_flags = UString(m_buffer16);
845 deleteAllValues(m_strings);
846 Vector<UString*> newStrings;
847 newStrings.reserveCapacity(initialStringTableCapacity);
848 m_strings.swap(newStrings);
850 deleteAllValues(m_identifiers);
851 Vector<KJS::Identifier*> newIdentifiers;
852 newIdentifiers.reserveCapacity(initialStringTableCapacity);
853 m_identifiers.swap(newIdentifiers);
855 Vector<char> newBuffer8;
856 newBuffer8.reserveCapacity(initialReadBufferCapacity);
857 m_buffer8.swap(newBuffer8);
859 Vector<UChar> newBuffer16;
860 newBuffer16.reserveCapacity(initialReadBufferCapacity);
861 m_buffer16.swap(newBuffer16);
867 Identifier* Lexer::makeIdentifier(const Vector<KJS::UChar>& buffer)
869 KJS::Identifier* identifier = new KJS::Identifier(buffer.data(), buffer.size());
870 m_identifiers.append(identifier);
874 UString* Lexer::makeUString(const Vector<KJS::UChar>& buffer)
876 UString* string = new UString(buffer);
877 m_strings.append(string);