Tokenizer.h (1465B)
1 // 2 // Copyright 2012 The ANGLE Project Authors. All rights reserved. 3 // Use of this source code is governed by a BSD-style license that can be 4 // found in the LICENSE file. 5 // 6 7 #ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ 8 #define COMPILER_PREPROCESSOR_TOKENIZER_H_ 9 10 #include "common/angleutils.h" 11 #include "compiler/preprocessor/Input.h" 12 #include "compiler/preprocessor/Lexer.h" 13 14 namespace angle 15 { 16 17 namespace pp 18 { 19 20 class Diagnostics; 21 22 class Tokenizer : public Lexer 23 { 24 public: 25 struct Context 26 { 27 Diagnostics *diagnostics; 28 29 Input input; 30 // The location where yytext points to. Token location should track 31 // scanLoc instead of Input::mReadLoc because they may not be the same 32 // if text is buffered up in the scanner input buffer. 33 Input::Location scanLoc; 34 35 bool leadingSpace; 36 bool lineStart; 37 }; 38 39 Tokenizer(Diagnostics *diagnostics); 40 ~Tokenizer() override; 41 42 bool init(size_t count, const char *const string[], const int length[]); 43 44 void setFileNumber(int file); 45 void setLineNumber(int line); 46 void setMaxTokenSize(size_t maxTokenSize); 47 48 void lex(Token *token) override; 49 50 private: 51 bool initScanner(); 52 void destroyScanner(); 53 54 void *mHandle; // Scanner handle. 55 Context mContext; // Scanner extra. 56 size_t mMaxTokenSize; // Maximum token size 57 }; 58 59 } // namespace pp 60 61 } // namespace angle 62 63 #endif // COMPILER_PREPROCESSOR_TOKENIZER_H_