Changeset 846 for trunk/tools/qdoc3/tokenizer.cpp

Timestamp:

May 5, 2011, 5:36:53 AM (14 years ago)

Author:

Dmitry A. Kuminov

Message:

trunk: Merged in qt 4.7.2 sources from branches/vendor/nokia/qt.

Location:

trunk

Files:

• . (modified) (1 prop)
• tools/qdoc3/tokenizer.cpp (modified) (7 diffs)

trunk/tools/qdoc3/tokenizer.cpp

@@ -1 +1 @@
 /****************************************************************************
 **
-** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
 ** All rights reserved.
 ** Contact: Nokia Corporation (qt-info@nokia.com)
@@ -48 +48 @@
 #include <qregexp.h>
 #include <qstring.h>
+#include <qtextcodec.h>
 
 #include <ctype.h>
@@ -67 +68 @@
     "slots", "static", "struct", "template", "typedef", "typename",
     "union", "unsigned", "using", "virtual", "void", "volatile",
-    "__int64", "Q_OBJECT", "Q_OVERRIDE", "Q_PROPERTY",
+    "__int64",
+    "Q_OBJECT",
+    "Q_OVERRIDE",
+    "Q_PROPERTY",
+    "Q_PRIVATE_PROPERTY",
     "Q_DECLARE_SEQUENTIAL_ITERATOR",
     "Q_DECLARE_MUTABLE_SEQUENTIAL_ITERATOR",
@@ -98 +103 @@
 static QRegExp *falsehoods = 0;
 
+static QTextCodec *sourceCodec = 0;
+
 /*
   This function is a perfect hash function for the 37 keywords of C99
@@ -119 +126 @@
 }
 
-Tokenizer::Tokenizer(const Location& loc, FILE *in)
+Tokenizer::Tokenizer(const Location& loc, QFile &in)
 {
     init();
-    QFile file;
-    file.open(in, QIODevice::ReadOnly);
-    yyIn = file.readAll();
-    file.close();
+    yyIn = in.readAll();
     yyPos = 0;
     start(loc);
@@ -483 +487 @@
 {
     QString versionSym = config.getString(CONFIG_VERSIONSYM);
+
+    QString sourceEncoding = config.getString(CONFIG_SOURCEENCODING);
+    if (sourceEncoding.isEmpty())
+        sourceEncoding = QLatin1String("ISO-8859-1");
+    sourceCodec = QTextCodec::codecForName(sourceEncoding.toLocal8Bit());
 
     comment = new QRegExp("/(?:\\*.*\\*/|/.*\n|/[^\n]*$)");
@@ -751 +760 @@
 }
 
+QString Tokenizer::lexeme() const
+{
+    return sourceCodec->toUnicode(yyLex);
+}
+
+QString Tokenizer::previousLexeme() const
+{
+    return sourceCodec->toUnicode(yyPrevLex);
+}
+
 QT_END_NAMESPACE