2 This file is part of the KDE libraries
4 Copyright (C) 1999 Lars Knoll (knoll@mpi-hd.mpg.de)
5 Copyright (C) 2003, 2004, 2005, 2006 Apple Computer, Inc.
6 Copyright (C) 2005, 2006 Alexey Proskuryakov (ap@nypop.com)
8 This library is free software; you can redistribute it and/or
9 modify it under the terms of the GNU Library General Public
10 License as published by the Free Software Foundation; either
11 version 2 of the License, or (at your option) any later version.
13 This library is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Library General Public License for more details.
18 You should have received a copy of the GNU Library General Public License
19 along with this library; see the file COPYING.LIB. If not, write to
20 the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
21 Boston, MA 02111-1307, USA.
29 #include "DOMImplementation.h"
30 #include "HTMLNames.h"
31 #include "StreamingTextDecoder.h"
32 #include "RegularExpression.h"
34 using namespace WebCore;
35 using namespace HTMLNames;
40 enum Type { ASCII, JIS, EUC, SJIS, UTF16, UTF8 };
41 static enum Type judge(const char *str, int length);
43 static const int _SS2_;
44 static const unsigned char kanji_map_sjis[];
45 static int ISkanji(int code)
49 return (kanji_map_sjis[code & 0xff] & 1);
52 static int ISkana(int code)
56 return (kanji_map_sjis[code & 0xff] & 2);
61 const int KanjiCode::ESC = 0x1b;
62 const int KanjiCode::_SS2_ = 0x8e;
64 const unsigned char KanjiCode::kanji_map_sjis[] =
66 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
67 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
68 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
69 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
70 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
71 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
72 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
73 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
74 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
75 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
76 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
77 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
78 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
79 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
80 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
81 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0
86 * [0xa1 - 0xfe][0xa1 - 0xfe]
87 * 0x8e[0xa1 - 0xfe](SS2)
88 * 0x8f[0xa1 - 0xfe][0xa1 - 0xfe](SS3)
91 * [0x81 - 0x9f, 0xe0 - 0xef(0xfe?)][0x40 - 0x7e, 0x80 - 0xfc]
93 * Shift_Jis Hankaku Kana is
98 * KanjiCode::judge() is based on judge_jcode() from jvim
99 * http://hp.vector.co.jp/authors/VA003457/vim/
101 * Special Thanks to Kenichi Tsuchida
104 enum KanjiCode::Type KanjiCode::judge(const char *str, int size)
108 int bfr = false; /* Kana Moji */
109 int bfk = 0; /* EUC Kana */
113 const unsigned char *ptr = (const unsigned char *) str;
119 if (ptr[i] == ESC && (size - i >= 3)) {
120 if ((ptr[i + 1] == '$' && ptr[i + 2] == 'B')
121 || (ptr[i + 1] == '(' && ptr[i + 2] == 'B')) {
124 } else if ((ptr[i + 1] == '$' && ptr[i + 2] == '@')
125 || (ptr[i + 1] == '(' && ptr[i + 2] == 'J')) {
128 } else if (ptr[i + 1] == '(' && ptr[i + 2] == 'I') {
131 } else if (ptr[i + 1] == ')' && ptr[i + 2] == 'I') {
143 /* ?? check kudokuten ?? && ?? hiragana ?? */
144 if ((i >= 2) && (ptr[i - 2] == 0x81)
145 && (0x41 <= ptr[i - 1] && ptr[i - 1] <= 0x49)) {
147 sjis += 100; /* kudokuten */
148 } else if ((i >= 2) && (ptr[i - 2] == 0xa1)
149 && (0xa2 <= ptr[i - 1] && ptr[i - 1] <= 0xaa)) {
151 euc += 100; /* kudokuten */
152 } else if ((i >= 2) && (ptr[i - 2] == 0x82) && (0xa0 <= ptr[i - 1])) {
153 sjis += 40; /* hiragana */
154 } else if ((i >= 2) && (ptr[i - 2] == 0xa4) && (0xa0 <= ptr[i - 1])) {
155 euc += 40; /* hiragana */
158 /* ?? check hiragana or katana ?? */
159 if ((size - i > 1) && (ptr[i] == 0x82) && (0xa0 <= ptr[i + 1])) {
160 sjis++; /* hiragana */
161 } else if ((size - i > 1) && (ptr[i] == 0x83)
162 && (0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x9f)) {
163 sjis++; /* katakana */
164 } else if ((size - i > 1) && (ptr[i] == 0xa4) && (0xa0 <= ptr[i + 1])) {
165 euc++; /* hiragana */
166 } else if ((size - i > 1) && (ptr[i] == 0xa5) && (0xa0 <= ptr[i + 1])) {
167 euc++; /* katakana */
170 if ((i >= 1) && (0x40 <= ptr[i] && ptr[i] <= 0xa0) && ISkanji(ptr[i - 1])) {
173 } else if ((i >= 1) && (0x81 <= ptr[i - 1] && ptr[i - 1] <= 0x9f) && ((0x40 <= ptr[i] && ptr[i] < 0x7e) || (0x7e < ptr[i] && ptr[i] <= 0xfc))) {
176 } else if ((i >= 1) && (0xfd <= ptr[i] && ptr[i] <= 0xfe) && (0xa1 <= ptr[i - 1] && ptr[i - 1] <= 0xfe)) {
179 } else if ((i >= 1) && (0xfd <= ptr[i - 1] && ptr[i - 1] <= 0xfe) && (0xa1 <= ptr[i] && ptr[i] <= 0xfe)) {
182 } else if ((i >= 1) && (ptr[i] < 0xa0 || 0xdf < ptr[i]) && (0x8e == ptr[i - 1])) {
185 } else if (ptr[i] <= 0x7f) {
189 if (0xa1 <= ptr[i] && ptr[i] <= 0xa6) {
190 euc++; /* sjis hankaku kana kigo */
191 } else if (0xa1 <= ptr[i] && ptr[i] <= 0xdf) {
192 ; /* sjis hankaku kana */
193 } else if (0xa1 <= ptr[i] && ptr[i] <= 0xfe) {
195 } else if (0x8e == ptr[i]) {
197 } else if (0x20 <= ptr[i] && ptr[i] <= 0x7f) {
203 } else if (0x8e == ptr[i]) {
206 } else if (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xdf) {
207 /* EUC KANA or SJIS KANJI */
218 } else if (0x81 <= ptr[i] && ptr[i] <= 0x9f) {
222 && ((0x40 <= ptr[i + 1] && ptr[i + 1] <= 0x7e)
223 || (0x80 <= ptr[i + 1] && ptr[i + 1] <= 0xfc))) {
226 } else if (0xfd <= ptr[i] && ptr[i] <= 0xfe) {
230 && (0xa1 <= ptr[i + 1] && ptr[i + 1] <= 0xfe)) {
233 } else if (ptr[i] <= 0x7f) {
246 } else if (sjis < euc) {
254 Decoder::Decoder(const String& mimeType, const String& defaultEncodingName)
255 : m_encoding(defaultEncodingName.isNull() ? "iso8859-1" : defaultEncodingName.latin1())
256 , m_encodingName(defaultEncodingName.isNull() ? "iso8859-1" : defaultEncodingName.latin1())
257 , m_type(DefaultEncoding)
258 , m_reachedBody(false)
259 , m_checkedForCSSCharset(false)
260 , m_checkedForBOM(false)
262 if (mimeType == "text/css")
264 else if (mimeType == "text/html")
265 m_contentType = HTML;
266 else if (DOMImplementation::isXMLMIMEType(mimeType)) {
268 // Despite 8.5 "Text/xml with Omitted Charset" of RFC 3023, we do not assume us-ascii
269 // for text/xml, to match Firefox.
270 m_encoding = TextEncoding(UTF8Encoding);
271 m_encodingName = "UTF-8";
273 m_contentType = PlainText;
275 m_decoder.set(StreamingTextDecoder::create(m_encoding));
282 void Decoder::setEncodingName(const char* encodingName, EncodingSource type)
284 m_encodingName = encodingName;
285 m_encodingName = m_encodingName.lower();
287 if (m_encodingName.isEmpty())
290 bool eightBitOnly = type == EncodingFromMetaTag || type == EncodingFromXMLHeader || type == EncodingFromCSSCharset;
291 TextEncoding encoding = TextEncoding(m_encodingName, eightBitOnly);
293 // in case the encoding didn't exist, we keep the old one (fixes some sites specifying invalid encodings)
294 if (encoding.isValid()) {
295 m_encodingName = encoding.name();
296 m_encoding = encoding;
298 m_decoder.set(StreamingTextDecoder::create(m_encoding));
302 const char* Decoder::encodingName() const
304 return m_encodingName;
307 // Other browsers allow comments in the head section, so we need to also.
308 // It's important not to look for tags inside the comments.
309 static void skipComment(const char *&ptr, const char *pEnd)
312 // Allow <!-->; other browsers do.
318 // This is the real end of comment, "-->".
319 if (p[1] == '-' && p[2] == '>') {
323 // This is the incorrect end of comment that other browsers allow, "--!>".
324 if (p[1] == '-' && p[2] == '!' && p[3] == '>') {
335 // Returns the position of the encoding string.
336 static int findXMLEncoding(const DeprecatedCString &str, int &encodingLength)
338 int len = str.length();
340 int pos = str.find("encoding");
345 // Skip spaces and stray control characters.
346 while (str[pos] <= ' ' && pos != len)
354 // Skip spaces and stray control characters.
355 while (str[pos] <= ' ' && pos != len)
358 // Skip quotation mark.
359 char quoteMark = str[pos];
360 if (quoteMark != '"' && quoteMark != '\'')
364 // Find the trailing quotation mark.
366 while (str[end] != quoteMark)
372 encodingLength = end - pos;
376 // true if there is more to parse
377 static inline bool skipWhitespace(const char*& pos, const char* dataEnd)
379 while (pos < dataEnd && (*pos == '\t' || *pos == ' '))
381 return pos != dataEnd;
384 DeprecatedString Decoder::decode(const char *data, int len)
386 // Check for UTF-16 or UTF-8 BOM mark at the beginning, which is a sure sign of a Unicode encoding.
387 int bufferLength = m_buffer.length();
388 const int maximumBOMLength = 3;
389 if (!m_checkedForBOM && bufferLength + len >= maximumBOMLength) {
390 if (m_type != UserChosenEncoding) {
391 // Extract the first three bytes.
392 // Handle the case where some of bytes are already in the buffer.
393 // The last byte is always guaranteed to not be in the buffer.
394 const unsigned char *udata = (const unsigned char *)data;
395 unsigned char c1 = bufferLength >= 1 ? m_buffer[0].unicode() : *udata++;
396 unsigned char c2 = bufferLength >= 2 ? m_buffer[1].unicode() : *udata++;
397 ASSERT(bufferLength < 3);
398 unsigned char c3 = *udata;
400 // Check for the BOM.
401 const char *autoDetectedEncoding;
402 if ((c1 == 0xFE && c2 == 0xFF) || (c1 == 0xFF && c2 == 0xFE)) {
403 autoDetectedEncoding = "ISO-10646-UCS-2";
404 } else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF) {
405 autoDetectedEncoding = "UTF-8";
407 autoDetectedEncoding = 0;
410 // If we found a BOM, use the encoding it implies.
411 if (autoDetectedEncoding != 0) {
412 m_type = AutoDetectedEncoding;
413 m_encoding = TextEncoding(autoDetectedEncoding);
414 ASSERT(m_encoding.isValid());
415 m_encodingName = m_encoding.name();
416 m_decoder.set(StreamingTextDecoder::create(m_encoding));
419 m_checkedForBOM = true;
422 bool currentChunkInBuffer = false;
424 if (m_type == DefaultEncoding && m_contentType == CSS && !m_checkedForCSSCharset) {
425 m_buffer.append(data, len);
426 currentChunkInBuffer = true;
428 if (len > 8) { // strlen("@charset") == 8
429 const char* dataStart = m_buffer.latin1();
430 const char* dataEnd = dataStart + m_buffer.length();
432 if (dataStart[0] == '@' && dataStart[1] == 'c' && dataStart[2] == 'h' && dataStart[3] == 'a' && dataStart[4] == 'r' &&
433 dataStart[5] == 's' && dataStart[6] == 'e' && dataStart[7] == 't') {
436 const char* pos = dataStart;
437 if (!skipWhitespace(pos, dataEnd))
438 return DeprecatedString::null;
440 if (*pos == '"' || *pos == '\'') {
441 char quotationMark = *pos;
445 while (pos < dataEnd && *pos != quotationMark)
448 return DeprecatedString::null;
450 DeprecatedCString encodingName(dataStart, pos - dataStart + 1);
453 if (!skipWhitespace(pos, dataEnd))
454 return DeprecatedString::null;
457 setEncodingName(encodingName, EncodingFromCSSCharset);
460 m_checkedForCSSCharset = true;
462 return DeprecatedString::null;
464 } else if (m_type == DefaultEncoding && m_contentType != PlainText && !m_reachedBody) { // HTML and XML
465 // this is not completely efficient, since the function might go
466 // through the html head several times...
468 m_buffer.append(data, len);
469 currentChunkInBuffer = true;
471 // we still don't have an encoding, and are in the head
472 // the following tags are allowed in <head>:
473 // SCRIPT|STYLE|META|LINK|OBJECT|TITLE|BASE
475 // We stop scanning when a tag that is not permitted in <head>
476 // is seen, rather when </head> is seen, because that more closely
477 // matches behavior in other browsers; more details in
478 // <http://bugzilla.opendarwin.org/show_bug.cgi?id=3590>.
480 // Additionally, we ignore things that looks like tags in <title>; see
481 // <http://bugzilla.opendarwin.org/show_bug.cgi?id=4560>.
483 bool withinTitle = false;
485 const char *ptr = m_buffer.latin1();
486 const char *pEnd = ptr + m_buffer.length();
487 while (ptr != pEnd) {
493 if (ptr[0] == '!' && ptr[1] == '-' && ptr[2] == '-') {
495 skipComment(ptr, pEnd);
499 // Handle XML declaration, which can have encoding in it.
500 // This encoding is honored even for HTML documents.
501 if (ptr[0] == '?' && ptr[1] == 'x' && ptr[2] == 'm' && ptr[3] == 'l') {
502 const char *end = ptr;
503 while (*end != '>' && *end != '\0')
507 DeprecatedCString str(ptr, end - ptr);
509 int pos = findXMLEncoding(str, len);
511 setEncodingName(str.mid(pos, len), EncodingFromXMLHeader);
512 // continue looking for a charset - it may be specified in an HTTP-Equiv meta
513 } else if (ptr[0] == 0 && ptr[1] == '?' && ptr[2] == 0 && ptr[3] == 'x' && ptr[4] == 0 && ptr[5] == 'm' && ptr[6] == 0 && ptr[7] == 'l') {
514 // UTF-16 without BOM
515 setEncodingName(((ptr - m_buffer.latin1()) % 2) ? "UTF-16LE" : "UTF-16BE", AutoDetectedEncoding);
519 // the HTTP-EQUIV meta has no effect on XHTML
520 if (m_contentType == XML)
531 ((*ptr >= 'a') && (*ptr <= 'z') ||
532 (*ptr >= 'A') && (*ptr <= 'Z') ||
533 (*ptr >= '0') && (*ptr <= '9'))
536 tmp[len] = tolower(*ptr);
541 AtomicString tag(tmp);
546 if (!end && tag == metaTag) {
547 const char* end = ptr;
548 while (*end != '>' && *end != '\0')
552 DeprecatedCString str(ptr, (end-ptr)+1);
555 while (pos < (int)str.length()) {
556 if ((pos = str.find("charset", pos, false)) == -1)
560 while (pos < (int)str.length() && str[pos] <= ' ')
562 if (pos == (int)str.length())
564 if (str[pos++] != '=')
566 while (pos < (int)str.length() &&
567 (str[pos] <= ' ') || str[pos] == '=' || str[pos] == '"' || str[pos] == '\'')
571 if (pos == (int)str.length())
573 unsigned endpos = pos;
574 while (endpos < str.length() &&
575 str[endpos] != ' ' && str[endpos] != '"' && str[endpos] != '\'' &&
576 str[endpos] != ';' && str[endpos] != '>')
578 setEncodingName(str.mid(pos, endpos-pos), EncodingFromMetaTag);
579 if (m_type == EncodingFromMetaTag)
582 if (endpos >= str.length() || str[endpos] == '/' || str[endpos] == '>')
587 } else if (tag != scriptTag && tag != noscriptTag && tag != styleTag &&
588 tag != linkTag && tag != metaTag && tag != objectTag &&
589 tag != titleTag && tag != baseTag &&
590 (end || tag != htmlTag) && !withinTitle &&
591 (tag != headTag) && isalpha(tmp[0])) {
592 m_reachedBody = true;
599 return DeprecatedString::null;
603 // Do the auto-detect if our default encoding is one of the Japanese ones.
604 if (m_type != UserChosenEncoding && m_type != AutoDetectedEncoding && m_encoding.isJapanese()) {
605 const char *autoDetectedEncoding;
606 switch (KanjiCode::judge(data, len)) {
608 autoDetectedEncoding = "jis7";
611 autoDetectedEncoding = "eucjp";
613 case KanjiCode::SJIS:
614 autoDetectedEncoding = "sjis";
617 autoDetectedEncoding = NULL;
620 if (autoDetectedEncoding)
621 setEncodingName(autoDetectedEncoding, AutoDetectedEncoding);
624 // If we still haven't found an encoding, assume latin1
625 // (this can happen if an empty name is passed from outside).
626 if (m_encodingName.isEmpty() || !m_encoding.isValid()) {
627 m_encodingName = "iso8859-1";
628 m_encoding = TextEncoding(Latin1Encoding);
630 m_decoder.set(StreamingTextDecoder::create(m_encoding));
632 DeprecatedString out;
634 if (!m_buffer.isEmpty()) {
635 if (!currentChunkInBuffer)
636 m_buffer.append(data, len);
637 out = m_decoder->toUnicode(m_buffer.latin1(), m_buffer.length());
638 m_buffer.truncate(0);
640 out = m_decoder->toUnicode(data, len);
645 DeprecatedString Decoder::flush() const
647 return m_decoder->toUnicode(m_buffer.latin1(), m_buffer.length(), true);
650 // -----------------------------------------------------------------------------