diff options
Diffstat (limited to 'include/clang/Lex/Lexer.h')
| -rw-r--r-- | include/clang/Lex/Lexer.h | 100 |
1 files changed, 55 insertions, 45 deletions
diff --git a/include/clang/Lex/Lexer.h b/include/clang/Lex/Lexer.h index 3a73147152afc..c2db4d357c517 100644 --- a/include/clang/Lex/Lexer.h +++ b/include/clang/Lex/Lexer.h @@ -39,11 +39,12 @@ class Lexer : public PreprocessorLexer { SourceLocation FileLoc; // Location for start of file. LangOptions Features; // Features enabled by this language (cache). bool Is_PragmaLexer; // True if lexer for _Pragma handling. + bool IsEofCodeCompletion; // True if EOF is treated as a code-completion. //===--------------------------------------------------------------------===// // Context-specific lexing flags set by the preprocessor. // - + /// ExtendedTokenMode - The lexer can optionally keep comments and whitespace /// and return them as tokens. This is used for -C and -CC modes, and /// whitespace preservation can be useful for some clients that want to lex @@ -52,7 +53,7 @@ class Lexer : public PreprocessorLexer { /// When this is set to 2 it returns comments and whitespace. When set to 1 /// it returns comments, when it is set to 0 it returns normal tokens only. unsigned char ExtendedTokenMode; - + //===--------------------------------------------------------------------===// // Context that changes as the file is lexed. // NOTE: any state that mutates when in raw mode must have save/restore code @@ -65,14 +66,14 @@ class Lexer : public PreprocessorLexer { // IsAtStartOfLine - True if the next lexed token should get the "start of // line" flag set on it. bool IsAtStartOfLine; - + Lexer(const Lexer&); // DO NOT IMPLEMENT void operator=(const Lexer&); // DO NOT IMPLEMENT friend class Preprocessor; - + void InitLexer(const char *BufStart, const char *BufPtr, const char *BufEnd); public: - + /// Lexer constructor - Create a new lexer object for the specified buffer /// with the specified preprocessor managing the lexing process. This lexer /// assumes that the associated file buffer and Preprocessor objects will @@ -84,21 +85,21 @@ public: /// range will outlive it, so it doesn't take ownership of it. Lexer(SourceLocation FileLoc, const LangOptions &Features, const char *BufStart, const char *BufPtr, const char *BufEnd); - + /// Lexer constructor - Create a new raw lexer object. This object is only /// suitable for calls to 'LexRawToken'. This lexer assumes that the text /// range will outlive it, so it doesn't take ownership of it. Lexer(FileID FID, const SourceManager &SM, const LangOptions &Features); - + /// Create_PragmaLexer: Lexer constructor - Create a new lexer object for /// _Pragma expansion. This has a variety of magic semantics that this method /// sets up. It returns a new'd Lexer that must be delete'd when done. - static Lexer *Create_PragmaLexer(SourceLocation SpellingLoc, + static Lexer *Create_PragmaLexer(SourceLocation SpellingLoc, SourceLocation InstantiationLocStart, SourceLocation InstantiationLocEnd, unsigned TokLen, Preprocessor &PP); - - + + /// getFeatures - Return the language features currently enabled. NOTE: this /// lexer modifies features as a file is parsed! const LangOptions &getFeatures() const { return Features; } @@ -108,7 +109,7 @@ public: /// the virtual location encodes where we should *claim* the characters came /// from. Currently this is only used by _Pragma handling. SourceLocation getFileLoc() const { return FileLoc; } - + /// Lex - Return the next token in the file. If this is the end of file, it /// return the tok::eof token. Return true if an error occurred and /// compilation should terminate, false if normal. This implicitly involves @@ -116,14 +117,14 @@ public: void Lex(Token &Result) { // Start a new token. Result.startToken(); - - // NOTE, any changes here should also change code after calls to + + // NOTE, any changes here should also change code after calls to // Preprocessor::HandleDirective if (IsAtStartOfLine) { Result.setFlag(Token::StartOfLine); IsAtStartOfLine = false; } - + // Get a token. Note that this may delete the current lexer if the end of // file is reached. LexTokenInternal(Result); @@ -131,11 +132,11 @@ public: /// isPragmaLexer - Returns true if this Lexer is being used to lex a pragma. bool isPragmaLexer() const { return Is_PragmaLexer; } - + /// IndirectLex - An indirect call to 'Lex' that can be invoked via /// the PreprocessorLexer interface. void IndirectLex(Token &Result) { Lex(Result); } - + /// LexFromRawLexer - Lex a token from a designated raw lexer (one with no /// associated preprocessor object. Return true if the 'next character to /// read' pointer points at the end of the lexer buffer, false otherwise. @@ -144,7 +145,7 @@ public: Lex(Result); // Note that lexing to the end of the buffer doesn't implicitly delete the // lexer when in raw mode. - return BufferPtr == BufferEnd; + return BufferPtr == BufferEnd; } /// isKeepWhitespaceMode - Return true if the lexer should return tokens for @@ -168,23 +169,32 @@ public: bool inKeepCommentMode() const { return ExtendedTokenMode > 0; } - + /// SetCommentRetentionMode - Change the comment retention mode of the lexer /// to the specified mode. This is really only useful when lexing in raw /// mode, because otherwise the lexer needs to manage this. - void SetCommentRetentionState(bool Mode) { + void SetCommentRetentionState(bool Mode) { assert(!isKeepWhitespaceMode() && "Can't play with comment retention state when retaining whitespace"); ExtendedTokenMode = Mode ? 1 : 0; } + + /// \brief Specify that end-of-file is to be considered a code-completion + /// token. + /// + /// When in this mode, the end-of-file token will be immediately preceded + /// by a code-completion token. + void SetEofIsCodeCompletion(bool Val = true) { + IsEofCodeCompletion = Val; + } const char *getBufferStart() const { return BufferStart; } - + /// ReadToEndOfLine - Read the rest of the current preprocessor line as an /// uninterpreted string. This switches the lexer out of directive mode. std::string ReadToEndOfLine(); - - + + /// Diag - Forwarding function for diagnostics. This translate a source /// position in the current buffer into a SourceLocation object for rendering. DiagnosticBuilder Diag(const char *Loc, unsigned DiagID) const; @@ -192,20 +202,20 @@ public: /// getSourceLocation - Return a source location identifier for the specified /// offset in the current file. SourceLocation getSourceLocation(const char *Loc, unsigned TokLen = 1) const; - + /// getSourceLocation - Return a source location for the next character in /// the current file. SourceLocation getSourceLocation() { return getSourceLocation(BufferPtr); } - + /// Stringify - Convert the specified string into a C string by escaping '\' /// and " characters. This does not add surrounding ""'s to the string. /// If Charify is true, this escapes the ' character instead of ". static std::string Stringify(const std::string &Str, bool Charify = false); - + /// Stringify - Convert the specified string into a C string by escaping '\' /// and " characters. This does not add surrounding ""'s to the string. static void Stringify(llvm::SmallVectorImpl<char> &Str); - + /// MeasureTokenLength - Relex the token at the specified location and return /// its length in bytes in the input file. If the token needs cleaning (e.g. /// includes a trigraph or an escaped newline) then this count includes bytes @@ -213,7 +223,7 @@ public: static unsigned MeasureTokenLength(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts); - + //===--------------------------------------------------------------------===// // Internal implementation interfaces. private: @@ -228,7 +238,7 @@ private: /// takes that range and assigns it to the token as its location and size. In /// addition, since tokens cannot overlap, this also updates BufferPtr to be /// TokEnd. - void FormTokenWithChars(Token &Result, const char *TokEnd, + void FormTokenWithChars(Token &Result, const char *TokEnd, tok::TokenKind Kind) { unsigned TokLen = TokEnd-BufferPtr; Result.setLength(TokLen); @@ -236,7 +246,7 @@ private: Result.setKind(Kind); BufferPtr = TokEnd; } - + /// isNextPPTokenLParen - Return 1 if the next unexpanded token will return a /// tok::l_paren token, 0 if it is something else and 2 if there are no more /// tokens in the buffer controlled by this lexer. @@ -245,7 +255,7 @@ private: //===--------------------------------------------------------------------===// // Lexer character reading interfaces. public: - + // This lexer is built on two interfaces for reading characters, both of which // automatically provide phase 1/2 translation. getAndAdvanceChar is used // when we know that we will be reading a character from the input buffer and @@ -260,7 +270,7 @@ public: // approach allows us to emit diagnostics for characters (e.g. warnings about // trigraphs), knowing that they only are emitted if the character is // consumed. - + /// isObviouslySimpleCharacter - Return true if the specified character is /// obviously the same in translation phase 1 and translation phase 3. This /// can return false for characters that end up being the same, but it will @@ -268,7 +278,7 @@ public: static bool isObviouslySimpleCharacter(char C) { return C != '?' && C != '\\'; } - + /// getAndAdvanceChar - Read a single 'character' from the specified buffer, /// advance over it, and return it. This is tricky in several cases. Here we /// just handle the trivial case and fall-back to the non-inlined @@ -277,13 +287,13 @@ public: // If this is not a trigraph and not a UCN or escaped newline, return // quickly. if (isObviouslySimpleCharacter(Ptr[0])) return *Ptr++; - + unsigned Size = 0; char C = getCharAndSizeSlow(Ptr, Size, &Tok); Ptr += Size; return C; } - + private: /// ConsumeChar - When a character (identified by PeekCharAndSize) is consumed /// and added to a given token, check to see if there are diagnostics that @@ -300,7 +310,7 @@ private: getCharAndSizeSlow(Ptr, Size, &Tok); return Ptr+Size; } - + /// getCharAndSize - Peek a single 'character' from the specified buffer, /// get its size, and return it. This is tricky in several cases. Here we /// just handle the trivial case and fall-back to the non-inlined @@ -312,16 +322,16 @@ private: Size = 1; return *Ptr; } - + Size = 0; return getCharAndSizeSlow(Ptr, Size); } - + /// getCharAndSizeSlow - Handle the slow/uncommon case of the getCharAndSize /// method. char getCharAndSizeSlow(const char *Ptr, unsigned &Size, Token *Tok = 0); public: - + /// getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever /// emit a warning. static inline char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size, @@ -332,30 +342,30 @@ public: Size = 1; return *Ptr; } - + Size = 0; return getCharAndSizeSlowNoWarn(Ptr, Size, Features); } - + /// getEscapedNewLineSize - Return the size of the specified escaped newline, /// or 0 if it is not an escaped newline. P[-1] is known to be a "\" on entry /// to this function. static unsigned getEscapedNewLineSize(const char *P); - + /// SkipEscapedNewLines - If P points to an escaped newline (or a series of /// them), skip over them and return the first non-escaped-newline found, /// otherwise return P. static const char *SkipEscapedNewLines(const char *P); private: - + /// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a /// diagnostic. static char getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, const LangOptions &Features); - + //===--------------------------------------------------------------------===// // Other lexer functions. - + // Helper functions to lex the remainder of a token of the specific type. void LexIdentifier (Token &Result, const char *CurPtr); void LexNumericConstant (Token &Result, const char *CurPtr); @@ -363,7 +373,7 @@ private: void LexAngledStringLiteral(Token &Result, const char *CurPtr); void LexCharConstant (Token &Result, const char *CurPtr); bool LexEndOfFile (Token &Result, const char *CurPtr); - + bool SkipWhitespace (Token &Result, const char *CurPtr); bool SkipBCPLComment (Token &Result, const char *CurPtr); bool SkipBlockComment (Token &Result, const char *CurPtr); |