diff options
Diffstat (limited to 'include/clang/Lex/TokenLexer.h')
| -rw-r--r-- | include/clang/Lex/TokenLexer.h | 111 |
1 files changed, 75 insertions, 36 deletions
diff --git a/include/clang/Lex/TokenLexer.h b/include/clang/Lex/TokenLexer.h index fdeed44d8f9b..b8b0beabf2ba 100644 --- a/include/clang/Lex/TokenLexer.h +++ b/include/clang/Lex/TokenLexer.h @@ -1,4 +1,4 @@ -//===--- TokenLexer.h - Lex from a token buffer -----------------*- C++ -*-===// +//===- TokenLexer.h - Lex from a token buffer -------------------*- C++ -*-===// // // The LLVM Compiler Infrastructure // @@ -15,29 +15,31 @@ #define LLVM_CLANG_LEX_TOKENLEXER_H #include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/ArrayRef.h" namespace clang { - class MacroInfo; - class Preprocessor; - class Token; - class MacroArgs; + +class MacroArgs; +class MacroInfo; +class Preprocessor; +class Token; +class VAOptExpansionContext; /// TokenLexer - This implements a lexer that returns tokens from a macro body /// or token stream instead of lexing from a character buffer. This is used for /// macro expansion and _Pragma handling, for example. -/// class TokenLexer { + friend class Preprocessor; + /// Macro - The macro we are expanding from. This is null if expanding a /// token stream. - /// - MacroInfo *Macro; + MacroInfo *Macro = nullptr; /// ActualArgs - The actual arguments specified for a function-like macro, or /// null. The TokenLexer owns the pointed-to object. - MacroArgs *ActualArgs; + MacroArgs *ActualArgs = nullptr; /// PP - The current preprocessor object we are expanding for. - /// Preprocessor &PP; /// Tokens - This is the pointer to an array of tokens that the macro is @@ -49,15 +51,12 @@ class TokenLexer { /// Note that if it points into Preprocessor's cache buffer, the Preprocessor /// may update the pointer as needed. const Token *Tokens; - friend class Preprocessor; /// NumTokens - This is the length of the Tokens array. - /// unsigned NumTokens; - /// CurToken - This is the next token that Lex will return. - /// - unsigned CurToken; + /// This is the index of the next token that Lex will return. + unsigned CurTokenIdx; /// ExpandLocStart/End - The source location range where this macro was /// expanded. @@ -73,6 +72,7 @@ class TokenLexer { /// \brief Location of the macro definition. SourceLocation MacroDefStart; + /// \brief Length of the macro definition. unsigned MacroDefLength; @@ -99,8 +99,6 @@ class TokenLexer { /// should not be subject to further macro expansion. bool DisableMacroExpansion : 1; - TokenLexer(const TokenLexer &) = delete; - void operator=(const TokenLexer &) = delete; public: /// Create a TokenLexer for the specified macro with the specified actual /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. @@ -108,26 +106,30 @@ public: /// identifier for an object-like macro. TokenLexer(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs, Preprocessor &pp) - : Macro(nullptr), ActualArgs(nullptr), PP(pp), OwnsTokens(false) { + : PP(pp), OwnsTokens(false) { Init(Tok, ILEnd, MI, ActualArgs); } - /// Init - Initialize this TokenLexer to expand from the specified macro - /// with the specified argument information. Note that this ctor takes - /// ownership of the ActualArgs pointer. ILEnd specifies the location of the - /// ')' for a function-like macro or the identifier for an object-like macro. - void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, - MacroArgs *ActualArgs); - /// Create a TokenLexer for the specified token stream. If 'OwnsTokens' is /// specified, this takes ownership of the tokens and delete[]'s them when /// the token lexer is empty. TokenLexer(const Token *TokArray, unsigned NumToks, bool DisableExpansion, bool ownsTokens, Preprocessor &pp) - : Macro(nullptr), ActualArgs(nullptr), PP(pp), OwnsTokens(false) { + : PP(pp), OwnsTokens(false) { Init(TokArray, NumToks, DisableExpansion, ownsTokens); } + TokenLexer(const TokenLexer &) = delete; + TokenLexer &operator=(const TokenLexer &) = delete; + ~TokenLexer() { destroy(); } + + /// Init - Initialize this TokenLexer to expand from the specified macro + /// with the specified argument information. Note that this ctor takes + /// ownership of the ActualArgs pointer. ILEnd specifies the location of the + /// ')' for a function-like macro or the identifier for an object-like macro. + void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, + MacroArgs *ActualArgs); + /// Init - Initialize this TokenLexer with the specified token stream. /// This does not take ownership of the specified token vector. /// @@ -136,8 +138,6 @@ public: void Init(const Token *TokArray, unsigned NumToks, bool DisableMacroExpansion, bool OwnsTokens); - ~TokenLexer() { destroy(); } - /// isNextTokenLParen - If the next token lexed will pop this macro off the /// expansion stack, return 2. If the next unexpanded token is a '(', return /// 1, otherwise return 0. @@ -156,15 +156,54 @@ private: /// isAtEnd - Return true if the next lex call will pop this macro off the /// include stack. bool isAtEnd() const { - return CurToken == NumTokens; + return CurTokenIdx == NumTokens; } - /// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## - /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there - /// are is another ## after it, chomp it iteratively. Return the result as - /// Tok. If this returns true, the caller should immediately return the + /// Concatenates the next (sub-)sequence of \p Tokens separated by '##' + /// starting with LHSTok - stopping when we encounter a token that is neither + /// '##' nor preceded by '##'. Places the result back into \p LHSTok and sets + /// \p CurIdx to point to the token following the last one that was pasted. + /// + /// Also performs the MSVC extension wide-literal token pasting involved with: + /// \code L #macro-arg. \endcode + /// + /// \param[in,out] LHSTok - Contains the token to the left of '##' in \p + /// Tokens upon entry and will contain the resulting concatenated Token upon + /// exit. + /// + /// \param[in] TokenStream - The stream of Tokens we are lexing from. + /// + /// \param[in,out] CurIdx - Upon entry, \pTokens[\pCurIdx] must equal '##' + /// (with the exception of the MSVC extension mentioned above). Upon exit, it + /// is set to the index of the token following the last token that was + /// concatenated together. + /// + /// \returns If this returns true, the caller should immediately return the /// token. - bool PasteTokens(Token &Tok); + bool pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream, + unsigned int &CurIdx); + + /// Calls pasteTokens above, passing in the '*this' object's Tokens and + /// CurTokenIdx data members. + bool pasteTokens(Token &Tok); + + + /// Takes the tail sequence of tokens within ReplacementToks that represent + /// the just expanded __VA_OPT__ tokens (possibly zero tokens) and transforms + /// them into a string. \p VCtx is used to determine which token represents + /// the first __VA_OPT__ replacement token. + /// + /// \param[in,out] ReplacementToks - Contains the current Replacement Tokens + /// (prior to rescanning and token pasting), the tail end of which represents + /// the tokens just expanded through __VA_OPT__ processing. These (sub) + /// sequence of tokens are folded into one stringified token. + /// + /// \param[in] VCtx - contains relevent contextual information about the + /// state of the tokens around and including the __VA_OPT__ token, necessary + /// for stringification. + void stringifyVAOPTContents(SmallVectorImpl<Token> &ReplacementToks, + const VAOptExpansionContext &VCtx, + SourceLocation VAOPTClosingParenLoc); /// Expand the arguments of a function-like macro so that we can quickly /// return preexpanded tokens from Tokens. @@ -200,6 +239,6 @@ private: void PropagateLineStartLeadingSpaceInfo(Token &Result); }; -} // end namespace clang +} // namespace clang -#endif +#endif // LLVM_CLANG_LEX_TOKENLEXER_H |