source: trunk/tools/qdoc3/tokenizer.h@ 674

Last change on this file since 674 was 651, checked in by Dmitry A. Kuminov, 16 years ago

trunk: Merged in qt 4.6.2 sources.
File size: 6.1 KB
Line 
1/****************************************************************************
2**
3** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
4** All rights reserved.
5** Contact: Nokia Corporation (qt-info@nokia.com)
6**
7** This file is part of the tools applications of the Qt Toolkit.
8**
9** $QT_BEGIN_LICENSE:LGPL$
10** Commercial Usage
11** Licensees holding valid Qt Commercial licenses may use this file in
12** accordance with the Qt Commercial License Agreement provided with the
13** Software or, alternatively, in accordance with the terms contained in
14** a written agreement between you and Nokia.
15**
16** GNU Lesser General Public License Usage
17** Alternatively, this file may be used under the terms of the GNU Lesser
18** General Public License version 2.1 as published by the Free Software
19** Foundation and appearing in the file LICENSE.LGPL included in the
20** packaging of this file. Please review the following information to
21** ensure the GNU Lesser General Public License version 2.1 requirements
22** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
23**
24** In addition, as a special exception, Nokia gives you certain additional
25** rights. These rights are described in the Nokia Qt LGPL Exception
26** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
27**
28** GNU General Public License Usage
29** Alternatively, this file may be used under the terms of the GNU
30** General Public License version 3.0 as published by the Free Software
31** Foundation and appearing in the file LICENSE.GPL included in the
32** packaging of this file. Please review the following information to
33** ensure the GNU General Public License version 3.0 requirements will be
34** met: http://www.gnu.org/copyleft/gpl.html.
35**
36** If you have questions regarding the use of this file, please contact
37** Nokia at qt-info@nokia.com.
38** $QT_END_LICENSE$
39**
40****************************************************************************/
41
42/*
43 tokenizer.h
44*/
45
46#ifndef TOKENIZER_H
47#define TOKENIZER_H
48
49#include <qstack.h>
50#include <qstring.h>
51
52#include <stdio.h>
53
54#include "location.h"
55
56QT_BEGIN_NAMESPACE
57
58/*
59 Here come the C++ tokens we support. The first part contains
60 all-purpose tokens; then come keywords.
61
62 If you add a keyword, make sure to modify the keyword array in
63 tokenizer.cpp as well, and possibly adjust Tok_FirstKeyword and
64 Tok_LastKeyword.
65*/
66enum { Tok_Eoi, Tok_Ampersand, Tok_Aster, Tok_Caret, Tok_LeftParen,
67 Tok_RightParen, Tok_LeftParenAster, Tok_Equal, Tok_LeftBrace,
68 Tok_RightBrace, Tok_Semicolon, Tok_Colon, Tok_LeftAngle,
69 Tok_RightAngle, Tok_Comma, Tok_Ellipsis, Tok_Gulbrandsen,
70 Tok_LeftBracket, Tok_RightBracket, Tok_Tilde, Tok_SomeOperator,
71 Tok_Number, Tok_String, Tok_Doc, Tok_Comment, Tok_Ident, Tok_At,
72 Tok_char, Tok_class, Tok_const, Tok_double, Tok_enum,
73 Tok_explicit, Tok_friend, Tok_inline, Tok_int, Tok_long,
74 Tok_namespace, Tok_operator, Tok_private, Tok_protected,
75 Tok_public, Tok_short, Tok_signals, Tok_signed, Tok_slots,
76 Tok_static, Tok_struct, Tok_template, Tok_typedef,
77 Tok_typename, Tok_union, Tok_unsigned, Tok_using, Tok_virtual,
78 Tok_void, Tok_volatile, Tok_int64, Tok_Q_OBJECT, Tok_Q_OVERRIDE,
79 Tok_Q_PROPERTY, Tok_Q_DECLARE_SEQUENTIAL_ITERATOR,
80 Tok_Q_DECLARE_MUTABLE_SEQUENTIAL_ITERATOR,
81 Tok_Q_DECLARE_ASSOCIATIVE_ITERATOR,
82 Tok_Q_DECLARE_MUTABLE_ASSOCIATIVE_ITERATOR,
83 Tok_Q_DECLARE_FLAGS, Tok_Q_SIGNALS, Tok_Q_SLOTS, Tok_QT_COMPAT,
84 Tok_QT_COMPAT_CONSTRUCTOR, Tok_QT_DEPRECATED, Tok_QT_MOC_COMPAT,
85 Tok_QT_MODULE, Tok_QT3_SUPPORT, Tok_QT3_SUPPORT_CONSTRUCTOR,
86 Tok_QT3_MOC_SUPPORT, Tok_QDOC_PROPERTY,
87 Tok_FirstKeyword = Tok_char, Tok_LastKeyword = Tok_QDOC_PROPERTY };
88
89/*
90 The Tokenizer class implements lexical analysis of C++ source
91 files.
92
93 Not every operator or keyword of C++ is recognized; only those
94 that are interesting to us. Some Qt keywords or macros are also
95 recognized.
96*/
97
98class Tokenizer
99{
100 public:
101 Tokenizer(const Location& loc, const QByteArray &in);
102 Tokenizer(const Location& loc, FILE *in);
103
104 ~Tokenizer();
105
106 int getToken();
107 void setParsingFnOrMacro(bool macro) { parsingMacro = macro; }
108 bool parsingFnOrMacro() const { return parsingMacro; }
109
110 const Location &location() const { return yyTokLoc; }
111 QString previousLexeme() const { return QString(yyPrevLex); }
112 QString lexeme() const { return QString(yyLex); }
113 QString version() const { return yyVersion; }
114 int braceDepth() const { return yyBraceDepth; }
115 int parenDepth() const { return yyParenDepth; }
116 int bracketDepth() const { return yyBracketDepth; }
117
118 static void initialize(const Config &config);
119 static void terminate();
120 static bool isTrue(const QString &condition);
121
122 private:
123 void init();
124 void start(const Location& loc);
125 /*
126 This limit on the length of a lexeme seems fairly high, but a
127 doc comment can be arbitrarily long. The previous 65,536 limit
128 was reached by Mark Summerfield.
129 */
130 enum { yyLexBufSize = 524288 };
131
132 int getch()
133 {
134 return yyPos == yyIn.size() ? EOF : yyIn[yyPos++];
135 }
136
137 inline int getChar()
138 {
139 if (yyCh == EOF)
140 return EOF;
141 if (yyLexLen < yyLexBufSize - 1) {
142 yyLex[yyLexLen++] = (char) yyCh;
143 yyLex[yyLexLen] = '\0';
144 }
145 yyCurLoc.advance(yyCh);
146 int ch = getch();
147 if (ch == EOF)
148 return EOF;
149 // cast explicitely to make sure the value of ch
150 // is in range [0..255] to avoid assert messages
151 // when using debug CRT that checks its input.
152 return int(uint(uchar(ch)));
153 }
154
155 int getTokenAfterPreprocessor();
156 void pushSkipping(bool skip);
157 bool popSkipping();
158
159 Location yyTokLoc;
160 Location yyCurLoc;
161 char *yyLexBuf1;
162 char *yyLexBuf2;
163 char *yyPrevLex;
164 char *yyLex;
165 size_t yyLexLen;
166 QStack<bool> yyPreprocessorSkipping;
167 int yyNumPreprocessorSkipping;
168 int yyBraceDepth;
169 int yyParenDepth;
170 int yyBracketDepth;
171 int yyCh;
172
173 QString yyVersion;
174 bool parsingMacro;
175
176 protected:
177 QByteArray yyIn;
178 int yyPos;
179};
180
181QT_END_NAMESPACE
182
183#endif
Note: See TracBrowser for help on using the repository browser.