2 Copyright (C) 1997 Martin Jones (mjones@kde.org)
3 (C) 1997 Torben Weis (weis@kde.org)
4 (C) 1998 Waldo Bastian (bastian@kde.org)
5 (C) 2001 Dirk Mueller (mueller@kde.org)
6 Copyright (C) 2003, 2004, 2005, 2006 Apple Computer, Inc.
8 This library is free software; you can redistribute it and/or
9 modify it under the terms of the GNU Library General Public
10 License as published by the Free Software Foundation; either
11 version 2 of the License, or (at your option) any later version.
13 This library is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Library General Public License for more details.
18 You should have received a copy of the GNU Library General Public License
19 along with this library; see the file COPYING.LIB. If not, write to
20 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
21 Boston, MA 02111-1307, USA.
24 #ifndef HTMLTokenizer_h
25 #define HTMLTokenizer_h
27 #include "DeprecatedPtrQueue.h"
28 #include "NamedMappedAttrMap.h"
29 #include "SegmentedString.h"
31 #include "Tokenizer.h"
32 #include "CachedResourceClient.h"
33 #include <wtf/Vector.h>
34 #include <wtf/OwnPtr.h>
39 class DocumentFragment;
42 class HTMLViewSourceDocument;
49 * represents one HTML tag. Consists of a numerical id, and the list
50 * of attributes. Can also represent text. In this case the id = 0 and
51 * text contains the text.
55 Token() : beginTag(true), flat(false), brokenXMLStyle(false), m_sourceInfo(0) { }
58 void addAttribute(Document*, AtomicString& attrName, const AtomicString& v, bool viewSourceMode);
60 bool isOpenTag(const QualifiedName& fullName) const { return beginTag && fullName.localName() == tagName; }
61 bool isCloseTag(const QualifiedName& fullName) const { return !beginTag && fullName.localName() == tagName; }
70 brokenXMLStyle = false;
72 m_sourceInfo->clear();
75 void addViewSourceChar(UChar c) { if (!m_sourceInfo.get()) m_sourceInfo.set(new Vector<UChar>); m_sourceInfo->append(c); }
77 RefPtr<NamedMappedAttrMap> attrs;
78 RefPtr<StringImpl> text;
83 OwnPtr<Vector<UChar> > m_sourceInfo;
86 //-----------------------------------------------------------------------------
88 class HTMLTokenizer : public Tokenizer, public CachedResourceClient {
90 HTMLTokenizer(HTMLDocument*, bool reportErrors);
91 HTMLTokenizer(HTMLViewSourceDocument*);
92 HTMLTokenizer(DocumentFragment*);
93 virtual ~HTMLTokenizer();
95 virtual bool write(const SegmentedString&, bool appendData);
96 virtual void finish();
97 virtual void setForceSynchronous(bool force);
98 virtual bool isWaitingForScripts() const;
99 virtual void stopParsing();
100 virtual bool processingData() const;
101 virtual int executingScript() const { return m_executingScript; }
103 virtual int lineNumber() const { return lineno; }
104 virtual int columnNumber() const { return 1; }
106 int* lineNumberPtr() { return &lineno; }
108 bool processingContentWrittenByScript() const { return src.excludeLineNumbers(); }
113 // Where we are in parsing a tag
118 PassRefPtr<Node> processToken();
120 State processListing(SegmentedString, State);
121 State parseComment(SegmentedString&, State);
122 State parseServer(SegmentedString&, State);
123 State parseText(SegmentedString&, State);
124 State parseSpecial(SegmentedString&, State);
125 State parseTag(SegmentedString&, State);
126 State parseEntity(SegmentedString&, UChar*& dest, State, unsigned& _cBufferPos, bool start, bool parsingTag);
127 State parseProcessingInstruction(SegmentedString&, State);
128 State scriptHandler(State);
129 State scriptExecution(const DeprecatedString& script, State, DeprecatedString scriptURL, int baseLine = 0);
130 void setSrc(const SegmentedString&);
132 // check if we have enough space in the buffer.
134 inline void checkBuffer(int len = 10)
136 if ((dest - buffer) > size - len)
140 inline void checkScriptBuffer(int len = 10)
142 if (scriptCodeSize + len >= scriptCodeMaxSize)
143 enlargeScriptBuffer(len);
146 void enlargeBuffer(int len);
147 void enlargeScriptBuffer(int len);
149 bool continueProcessing(int& processedCount, double startTime, State&);
150 void timerFired(Timer<HTMLTokenizer>*);
151 void allDataProcessed();
153 // from CachedResourceClient
154 void notifyFinished(CachedResource *finishedObj);
163 // the size of buffer
168 // are we in quotes within a html tag
169 enum { NoQuote, SingleQuote, DoubleQuote } tquote;
171 // Are we in a &... character entity description?
181 unsigned EntityUnicodeValue;
197 State() : m_bits(0) { }
199 TagState tagState() const { return static_cast<TagState>(m_bits & TagMask); }
200 void setTagState(TagState t) { m_bits = (m_bits & ~TagMask) | t; }
201 EntityState entityState() const { return static_cast<EntityState>((m_bits & EntityMask) >> EntityShift); }
202 void setEntityState(EntityState e) { m_bits = (m_bits & ~EntityMask) | (e << EntityShift); }
204 bool inScript() const { return testBit(InScript); }
205 void setInScript(bool v) { setBit(InScript, v); }
206 bool inStyle() const { return testBit(InStyle); }
207 void setInStyle(bool v) { setBit(InStyle, v); }
208 bool inXmp() const { return testBit(InXmp); }
209 void setInXmp(bool v) { setBit(InXmp, v); }
210 bool inTitle() const { return testBit(InTitle); }
211 void setInTitle(bool v) { setBit(InTitle, v); }
212 bool inPlainText() const { return testBit(InPlainText); }
213 void setInPlainText(bool v) { setBit(InPlainText, v); }
214 bool inProcessingInstruction() const { return testBit(InProcessingInstruction); }
215 void setInProcessingInstruction(bool v) { return setBit(InProcessingInstruction, v); }
216 bool inComment() const { return testBit(InComment); }
217 void setInComment(bool v) { setBit(InComment, v); }
218 bool inTextArea() const { return testBit(InTextArea); }
219 void setInTextArea(bool v) { setBit(InTextArea, v); }
220 bool escaped() const { return testBit(Escaped); }
221 void setEscaped(bool v) { setBit(Escaped, v); }
222 bool inServer() const { return testBit(InServer); }
223 void setInServer(bool v) { setBit(InServer, v); }
224 bool skipLF() const { return testBit(SkipLF); }
225 void setSkipLF(bool v) { setBit(SkipLF, v); }
226 bool startTag() const { return testBit(StartTag); }
227 void setStartTag(bool v) { setBit(StartTag, v); }
228 bool discardLF() const { return testBit(DiscardLF); }
229 void setDiscardLF(bool v) { setBit(DiscardLF, v); }
230 bool allowYield() const { return testBit(AllowYield); }
231 void setAllowYield(bool v) { setBit(AllowYield, v); }
232 bool loadingExtScript() const { return testBit(LoadingExtScript); }
233 void setLoadingExtScript(bool v) { setBit(LoadingExtScript, v); }
234 bool forceSynchronous() const { return testBit(ForceSynchronous); }
235 void setForceSynchronous(bool v) { setBit(ForceSynchronous, v); }
237 bool inAnySpecial() const { return m_bits & (InScript | InStyle | InXmp | InTextArea | InTitle); }
238 bool hasTagState() const { return m_bits & TagMask; }
239 bool hasEntityState() const { return m_bits & EntityMask; }
241 bool needsSpecialWriteHandling() const { return m_bits & (InScript | InStyle | InXmp | InTextArea | InTitle | TagMask | EntityMask | InPlainText | InComment | InServer | InProcessingInstruction | StartTag); }
244 static const int EntityShift = 4;
246 TagMask = (1 << 4) - 1,
247 EntityMask = (1 << 7) - (1 << 4),
253 InPlainText = 1 << 12,
254 InProcessingInstruction = 1 << 13,
256 InTextArea = 1 << 15,
261 DiscardLF = 1 << 20, // FIXME: should clarify difference between skip and discard
262 AllowYield = 1 << 21,
263 LoadingExtScript = 1 << 22,
264 ForceSynchronous = 1 << 23
267 void setBit(StateBits bit, bool value)
274 bool testBit(StateBits bit) const { return m_bits & bit; }
283 // Name of an attribute that we just scanned.
284 AtomicString attrName;
286 // Used to store the code of a srcipting sequence
288 // Size of the script sequenze stored in @ref #scriptCode
290 // Maximal size that can be stored in @ref #scriptCode
291 int scriptCodeMaxSize;
292 // resync point of script code size
293 int scriptCodeResync;
295 // Stores characters if we are scanning for a string like "</script>"
296 UChar searchBuffer[10];
297 // Counts where we are in the string we are scanning for
299 // The string we are searching for
300 const UChar* searchFor;
301 // the stopper string
302 const char* searchStopper;
304 int searchStopperLen;
305 // if no more data is coming, just parse what we have (including ext scripts that
306 // may be still downloading) and finish
308 // URL to get source code of script from
310 String scriptSrcCharset;
311 // the HTML code we will parse after the external script we are waiting for has loaded
312 SegmentedString pendingSrc;
314 // the HTML code we will parse after this particular script has
315 // loaded, but before all pending HTML
316 SegmentedString *currentPrependingSrc;
318 // true if we are executing a script while parsing a document. This causes the parsing of
319 // the output of the script to be postponed until after the script has finished executing
320 int m_executingScript;
321 DeprecatedPtrQueue<CachedScript> pendingScripts;
322 RefPtr<Node> scriptNode;
324 bool m_requestingScript;
326 // if we found one broken comment, there are most likely others as well
327 // store a flag to get rid of the O(n^2) behaviour in such a case.
329 // current line number
331 // line number at which the current <script> started
332 int scriptStartLineno;
335 // The timer for continued processing.
336 Timer<HTMLTokenizer> m_timer;
338 // This buffer can hold arbitrarily long user-defined attribute names, such as in EMBED tags.
339 // So any fixed number might be too small, but rather than rewriting all usage of this buffer
340 // we'll just make it large enough to handle all imaginable cases.
342 char cBuffer[CBUFLEN + 2];
343 unsigned int m_cBufferPos;
352 void parseHTMLDocumentFragment(const String&, DocumentFragment*);
354 UChar decodeNamedEntity(const char*);
356 } // namespace WebCore
358 #endif // HTMLTokenizer_h