9 files changed, 314 insertions, 99 deletions

diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 52a7a04567a6..a91e40435cb0 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -33,7 +33,7 @@
 #include <cctype>
 using namespace clang;
 
-static void InitCharacterInfo();
+static void InitCharacterInfo(LangOptions);
 
 //===----------------------------------------------------------------------===//
 // Token Class Implementation
@@ -59,7 +59,7 @@ tok::ObjCKeywordKind Token::getObjCKeywordID() const {
 
 void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
                       const char *BufEnd) {
-  InitCharacterInfo();
+  InitCharacterInfo(Features);
 
   BufferStart = BufStart;
   BufferPtr = BufPtr;
@@ -70,7 +70,7 @@ void Lexer::InitLexer(const char *BufStart, const char *BufPtr,
          " to simplify lexing!");
 
   Is_PragmaLexer = false;
-  IsEofCodeCompletion = false;
+  IsInConflictMarker = false;
   
   // Start of the file is a start of line.
   IsAtStartOfLine = true;
@@ -105,10 +105,6 @@ Lexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP)
 
   // Default to keeping comments if the preprocessor wants them.
   SetCommentRetentionState(PP.getCommentRetentionState());
-      
-  // If the input file is truncated, the EOF is a code-completion token.
-  if (PP.getSourceManager().isTruncatedFile(FID))
-    IsEofCodeCompletion = true;
 }
 
 /// Lexer constructor - Create a new raw lexer object.  This object is only
@@ -258,7 +254,7 @@ enum {
 
 // Statically initialize CharInfo table based on ASCII character set
 // Reference: FreeBSD 7.2 /usr/share/misc/ascii
-static const unsigned char CharInfo[256] =
+static unsigned char CharInfo[256] =
 {
 // 0 NUL         1 SOH         2 STX         3 ETX
 // 4 EOT         5 ENQ         6 ACK         7 BEL
@@ -326,7 +322,7 @@ static const unsigned char CharInfo[256] =
    0           , 0           , 0           , 0
 };
 
-static void InitCharacterInfo() {
+static void InitCharacterInfo(LangOptions Features) {
   static bool isInited = false;
   if (isInited) return;
   // check the statically-initialized CharInfo table
@@ -344,6 +340,11 @@ static void InitCharacterInfo() {
   }
   for (unsigned i = '0'; i <= '9'; ++i)
     assert(CHAR_NUMBER == CharInfo[i]);
+    
+  if (Features.Microsoft)
+    // Hack to treat DOS & CP/M EOF (^Z) as horizontal whitespace.
+    CharInfo[26/*sub*/] = CHAR_HORZ_WS;  
+
   isInited = true;
 }
 
@@ -1326,24 +1327,16 @@ bool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) {
   // Otherwise, check if we are code-completing, then issue diagnostics for 
   // unterminated #if and missing newline.
 
-  if (IsEofCodeCompletion) {
-    bool isIntendedFile = true;
-    if (PP && FileLoc.isFileID()) {
-      SourceManager &SM = PP->getSourceManager();
-      isIntendedFile = SM.isTruncatedFile(SM.getFileID(FileLoc));
-    }
+  if (PP && PP->isCodeCompletionFile(FileLoc)) {
+    // We're at the end of the file, but we've been asked to consider the
+    // end of the file to be a code-completion token. Return the
+    // code-completion token.
+    Result.startToken();
+    FormTokenWithChars(Result, CurPtr, tok::code_completion);
     
-    if (isIntendedFile) {
-      // We're at the end of the file, but we've been asked to consider the
-      // end of the file to be a code-completion token. Return the
-      // code-completion token.
-      Result.startToken();
-      FormTokenWithChars(Result, CurPtr, tok::code_completion);
-      
-      // Only do the eof -> code_completion translation once.
-      IsEofCodeCompletion = false;
-      return true;
-    }
+    // Only do the eof -> code_completion translation once.
+    PP->SetCodeCompletionPoint(0, 0, 0);
+    return true;
   }
   
   // If we are in a #if directive, emit an error.
@@ -1398,6 +1391,105 @@ unsigned Lexer::isNextPPTokenLParen() {
   return Tok.is(tok::l_paren);
 }
 
+/// FindConflictEnd - Find the end of a version control conflict marker.
+static const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd) {
+  llvm::StringRef RestOfBuffer(CurPtr+7, BufferEnd-CurPtr-7);
+  size_t Pos = RestOfBuffer.find(">>>>>>>");
+  while (Pos != llvm::StringRef::npos) {
+    // Must occur at start of line.
+    if (RestOfBuffer[Pos-1] != '\r' &&
+        RestOfBuffer[Pos-1] != '\n') {
+      RestOfBuffer = RestOfBuffer.substr(Pos+7);
+      continue;
+    }
+    return RestOfBuffer.data()+Pos;
+  }
+  return 0;
+}
+
+/// IsStartOfConflictMarker - If the specified pointer is the start of a version
+/// control conflict marker like '<<<<<<<', recognize it as such, emit an error
+/// and recover nicely.  This returns true if it is a conflict marker and false
+/// if not.
+bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
+  // Only a conflict marker if it starts at the beginning of a line.
+  if (CurPtr != BufferStart &&
+      CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
+    return false;
+  
+  // Check to see if we have <<<<<<<.
+  if (BufferEnd-CurPtr < 8 ||
+      llvm::StringRef(CurPtr, 7) != "<<<<<<<")
+    return false;
+
+  // If we have a situation where we don't care about conflict markers, ignore
+  // it.
+  if (IsInConflictMarker || isLexingRawMode())
+    return false;
+  
+  // Check to see if there is a >>>>>>> somewhere in the buffer at the start of
+  // a line to terminate this conflict marker.
+  if (FindConflictEnd(CurPtr+7, BufferEnd)) {
+    // We found a match.  We are really in a conflict marker.
+    // Diagnose this, and ignore to the end of line.
+    Diag(CurPtr, diag::err_conflict_marker);
+    IsInConflictMarker = true;
+    
+    // Skip ahead to the end of line.  We know this exists because the
+    // end-of-conflict marker starts with \r or \n.
+    while (*CurPtr != '\r' && *CurPtr != '\n') {
+      assert(CurPtr != BufferEnd && "Didn't find end of line");
+      ++CurPtr;
+    }
+    BufferPtr = CurPtr;
+    return true;
+  }
+  
+  // No end of conflict marker found.
+  return false;
+}
+
+
+/// HandleEndOfConflictMarker - If this is a '=======' or '|||||||' or '>>>>>>>'
+/// marker, then it is the end of a conflict marker.  Handle it by ignoring up
+/// until the end of the line.  This returns true if it is a conflict marker and
+/// false if not.
+bool Lexer::HandleEndOfConflictMarker(const char *CurPtr) {
+  // Only a conflict marker if it starts at the beginning of a line.
+  if (CurPtr != BufferStart &&
+      CurPtr[-1] != '\n' && CurPtr[-1] != '\r')
+    return false;
+  
+  // If we have a situation where we don't care about conflict markers, ignore
+  // it.
+  if (!IsInConflictMarker || isLexingRawMode())
+    return false;
+  
+  // Check to see if we have the marker (7 characters in a row).
+  for (unsigned i = 1; i != 7; ++i)
+    if (CurPtr[i] != CurPtr[0])
+      return false;
+  
+  // If we do have it, search for the end of the conflict marker.  This could
+  // fail if it got skipped with a '#if 0' or something.  Note that CurPtr might
+  // be the end of conflict marker.
+  if (const char *End = FindConflictEnd(CurPtr, BufferEnd)) {
+    CurPtr = End;
+    
+    // Skip ahead to the end of line.
+    while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n')
+      ++CurPtr;
+    
+    BufferPtr = CurPtr;
+    
+    // No longer in the conflict marker.
+    IsInConflictMarker = false;
+    return true;
+  }
+  
+  return false;
+}
+
 
 /// LexTokenInternal - This implements a simple C family lexer.  It is an
 /// extremely performance critical piece of code.  This assumes that the buffer
@@ -1764,14 +1856,20 @@ LexNextToken:
     Char = getCharAndSize(CurPtr, SizeTmp);
     if (ParsingFilename) {
       return LexAngledStringLiteral(Result, CurPtr);
-    } else if (Char == '<' &&
-               getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '=') {
-      Kind = tok::lesslessequal;
-      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
-                           SizeTmp2, Result);
     } else if (Char == '<') {
-      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
-      Kind = tok::lessless;
+      char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
+      if (After == '=') {
+        Kind = tok::lesslessequal;
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
+      } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) {
+        // If this is actually a '<<<<<<<' version control conflict marker,
+        // recognize it as such and recover nicely.
+        goto LexNextToken;
+      } else {
+        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+        Kind = tok::lessless;
+      }
     } else if (Char == '=') {
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
       Kind = tok::lessequal;
@@ -1790,14 +1888,20 @@ LexNextToken:
     if (Char == '=') {
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
       Kind = tok::greaterequal;
-    } else if (Char == '>' &&
-               getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '=') {
-      CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
-                           SizeTmp2, Result);
-      Kind = tok::greatergreaterequal;
     } else if (Char == '>') {
-      CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
-      Kind = tok::greatergreater;
+      char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2);
+      if (After == '=') {
+        CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result),
+                             SizeTmp2, Result);
+        Kind = tok::greatergreaterequal;
+      } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) {
+        // If this is '>>>>>>>' and we're in a conflict marker, ignore it.
+        goto LexNextToken;
+      } else {
+        CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
+        Kind = tok::greatergreater;
+      }
+      
     } else {
       Kind = tok::greater;
     }
@@ -1817,6 +1921,9 @@ LexNextToken:
       Kind = tok::pipeequal;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else if (Char == '|') {
+      // If this is '|||||||' and we're in a conflict marker, ignore it.
+      if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1))
+        goto LexNextToken;
       Kind = tok::pipepipe;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
@@ -1841,6 +1948,10 @@ LexNextToken:
   case '=':
     Char = getCharAndSize(CurPtr, SizeTmp);
     if (Char == '=') {
+      // If this is '=======' and we're in a conflict marker, ignore it.
+      if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1))
+        goto LexNextToken;
+      
       Kind = tok::equalequal;
       CurPtr = ConsumeChar(CurPtr, SizeTmp, Result);
     } else {
diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp
index c14d7c438d60..376cce8eb321 100644
--- a/lib/Lex/MacroArgs.cpp
+++ b/lib/Lex/MacroArgs.cpp
@@ -20,7 +20,8 @@ using namespace clang;
 /// MacroArgs ctor function - This destroys the vector passed in.
 MacroArgs *MacroArgs::create(const MacroInfo *MI,
                              const Token *UnexpArgTokens,
-                             unsigned NumToks, bool VarargsElided) {
+                             unsigned NumToks, bool VarargsElided,
+                             Preprocessor &PP) {
   assert(MI->isFunctionLike() &&
          "Can't have args for an object-like macro!");
 
@@ -40,13 +41,26 @@ MacroArgs *MacroArgs::create(const MacroInfo *MI,
 
 /// destroy - Destroy and deallocate the memory for this object.
 ///
-void MacroArgs::destroy() {
+void MacroArgs::destroy(Preprocessor &PP) {
   // Run the dtor to deallocate the vectors.
   this->~MacroArgs();
   // Release the memory for the object.
   free(this);
 }
 
+/// deallocate - This should only be called by the Preprocessor when managing
+/// its freelist.
+MacroArgs *MacroArgs::deallocate() {
+  MacroArgs *Next = ArgCache;
+  
+  // Run the dtor to deallocate the vectors.
+  this->~MacroArgs();
+  // Release the memory for the object.
+  free(this);
+  
+  return Next;
+}
+
 
 /// getArgLength - Given a pointer to an expanded or unexpanded argument,
 /// return the number of tokens, not counting the EOF, that make up the
diff --git a/lib/Lex/MacroArgs.h b/lib/Lex/MacroArgs.h
index 8dee5b3bc997..fa040c7a4d6f 100644
--- a/lib/Lex/MacroArgs.h
+++ b/lib/Lex/MacroArgs.h
@@ -30,6 +30,13 @@ class MacroArgs {
   /// concatenated together, with 'EOF' markers at the end of each argument.
   unsigned NumUnexpArgTokens;
 
+  /// VarargsElided - True if this is a C99 style varargs macro invocation and
+  /// there was no argument specified for the "..." argument.  If the argument
+  /// was specified (even empty) or this isn't a C99 style varargs function, or
+  /// if in strict mode and the C99 varargs macro had only a ... argument, this
+  /// is false.
+  bool VarargsElided;
+  
   /// PreExpArgTokens - Pre-expanded tokens for arguments that need them.  Empty
   /// if not yet computed.  This includes the EOF marker at the end of the
   /// stream.
@@ -39,26 +46,24 @@ class MacroArgs {
   /// stringified form of an argument has not yet been computed, this is empty.
   std::vector<Token> StringifiedArgs;
 
-  /// VarargsElided - True if this is a C99 style varargs macro invocation and
-  /// there was no argument specified for the "..." argument.  If the argument
-  /// was specified (even empty) or this isn't a C99 style varargs function, or
-  /// if in strict mode and the C99 varargs macro had only a ... argument, this
-  /// is false.
-  bool VarargsElided;
-
+  /// ArgCache - This is a linked list of MacroArgs objects that the
+  /// Preprocessor owns which we use to avoid thrashing malloc/free.
+  MacroArgs *ArgCache;
+  
   MacroArgs(unsigned NumToks, bool varargsElided)
-    : NumUnexpArgTokens(NumToks), VarargsElided(varargsElided) {}
+    : NumUnexpArgTokens(NumToks), VarargsElided(varargsElided), ArgCache(0) {}
   ~MacroArgs() {}
 public:
   /// MacroArgs ctor function - Create a new MacroArgs object with the specified
   /// macro and argument info.
   static MacroArgs *create(const MacroInfo *MI,
                            const Token *UnexpArgTokens,
-                           unsigned NumArgTokens, bool VarargsElided);
+                           unsigned NumArgTokens, bool VarargsElided,
+                           Preprocessor &PP);
 
   /// destroy - Destroy and deallocate the memory for this object.
   ///
-  void destroy();
+  void destroy(Preprocessor &PP);
 
   /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
   /// by pre-expansion, return false.  Otherwise, conservatively return true.
@@ -102,6 +107,11 @@ public:
   ///
   static Token StringifyArgument(const Token *ArgToks,
                                  Preprocessor &PP, bool Charify = false);
+  
+  
+  /// deallocate - This should only be called by the Preprocessor when managing
+  /// its freelist.
+  MacroArgs *deallocate();
 };
 
 }  // end namespace clang
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 9caca339be32..f5c60eb49438 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -481,11 +481,11 @@ void Preprocessor::HandleDirective(Token &Result) {
   CurPPLexer->ParsingPreprocessorDirective = true;
 
   ++NumDirectives;
-
+  
   // We are about to read a token.  For the multiple-include optimization FA to
   // work, we have to remember if we had read any tokens *before* this
   // pp-directive.
-  bool ReadAnyTokensBeforeDirective = CurPPLexer->MIOpt.getHasReadAnyTokensVal();
+  bool ReadAnyTokensBeforeDirective =CurPPLexer->MIOpt.getHasReadAnyTokensVal();
 
   // Save the '#' token in case we need to return it later.
   Token SavedHash = Result;
@@ -1112,9 +1112,10 @@ void Preprocessor::HandleIncludeDirective(Token &IncludeTok,
   }
 
   // Finally, if all is good, enter the new file!
-  if (EnterSourceFile(FID, CurDir))
+  std::string ErrorStr;
+  if (EnterSourceFile(FID, CurDir, ErrorStr))
     Diag(FilenameTok, diag::err_pp_error_opening_file)
-      << std::string(SourceMgr.getFileEntryForID(FID)->getName());
+      << std::string(SourceMgr.getFileEntryForID(FID)->getName()) << ErrorStr;
 }
 
 /// HandleIncludeNextDirective - Implements #include_next.
@@ -1548,8 +1549,9 @@ void Preprocessor::HandleIfdefDirective(Token &Result, bool isIfndef,
   // Should we include the stuff contained by this directive?
   if (!MI == isIfndef) {
     // Yes, remember that we are inside a conditional, then lex the next token.
-    CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(), /*wasskip*/false,
-                                   /*foundnonskip*/true, /*foundelse*/false);
+    CurPPLexer->pushConditionalLevel(DirectiveTok.getLocation(),
+                                     /*wasskip*/false, /*foundnonskip*/true,
+                                     /*foundelse*/false);
   } else {
     // No, skip the contents of this block and return the first token after it.
     SkipExcludedConditionalBlock(DirectiveTok.getLocation(),
diff --git a/lib/Lex/PPExpressions.cpp b/lib/Lex/PPExpressions.cpp
index b54dfe093b2c..2a6b2a729417 100644
--- a/lib/Lex/PPExpressions.cpp
+++ b/lib/Lex/PPExpressions.cpp
@@ -72,8 +72,8 @@ struct DefinedTracker {
 };
 
 /// EvaluateDefined - Process a 'defined(sym)' expression.
-static bool EvaluateDefined(PPValue &Result, Token &PeekTok,
-        DefinedTracker &DT, bool ValueLive, Preprocessor &PP) {
+static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
+                            bool ValueLive, Preprocessor &PP) {
   IdentifierInfo *II;
   Result.setBegin(PeekTok.getLocation());
 
@@ -142,22 +142,21 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
   // 'defined' or if it is a macro.  Note that we check here because many
   // keywords are pp-identifiers, so we can't check the kind.
   if (IdentifierInfo *II = PeekTok.getIdentifierInfo()) {
-    if (II->isStr("defined")) {
-      // Handle "defined X" and "defined(X)".
+    // Handle "defined X" and "defined(X)".
+    if (II->isStr("defined"))
       return(EvaluateDefined(Result, PeekTok, DT, ValueLive, PP));
-    } else {
-      // If this identifier isn't 'defined' or one of the special
-      // preprocessor keywords and it wasn't macro expanded, it turns
-      // into a simple 0, unless it is the C++ keyword "true", in which case it
-      // turns into "1".
-      if (ValueLive)
-        PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II;
-      Result.Val = II->getTokenID() == tok::kw_true;
-      Result.Val.setIsUnsigned(false);  // "0" is signed intmax_t 0.
-      Result.setRange(PeekTok.getLocation());
-      PP.LexNonComment(PeekTok);
-      return false;
-    }
+    
+    // If this identifier isn't 'defined' or one of the special
+    // preprocessor keywords and it wasn't macro expanded, it turns
+    // into a simple 0, unless it is the C++ keyword "true", in which case it
+    // turns into "1".
+    if (ValueLive)
+      PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II;
+    Result.Val = II->getTokenID() == tok::kw_true;
+    Result.Val.setIsUnsigned(false);  // "0" is signed intmax_t 0.
+    Result.setRange(PeekTok.getLocation());
+    PP.LexNonComment(PeekTok);
+    return false;
   }
 
   switch (PeekTok.getKind()) {
@@ -677,6 +676,15 @@ static bool EvaluateDirectiveSubExpr(PPValue &LHS, unsigned MinPrec,
 /// to "!defined(X)" return X in IfNDefMacro.
 bool Preprocessor::
 EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
+  // Save the current state of 'DisableMacroExpansion' and reset it to false. If
+  // 'DisableMacroExpansion' is true, then we must be in a macro argument list
+  // in which case a directive is undefined behavior.  We want macros to be able
+  // to recursively expand in order to get more gcc-list behavior, so we force
+  // DisableMacroExpansion to false and restore it when we're done parsing the
+  // expression.
+  bool DisableMacroExpansionAtStartOfDirective = DisableMacroExpansion;
+  DisableMacroExpansion = false;
+  
   // Peek ahead one token.
   Token Tok;
   Lex(Tok);
@@ -690,6 +698,9 @@ EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
     // Parse error, skip the rest of the macro line.
     if (Tok.isNot(tok::eom))
       DiscardUntilEndOfDirective();
+    
+    // Restore 'DisableMacroExpansion'.
+    DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
     return false;
   }
 
@@ -702,6 +713,8 @@ EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
     if (DT.State == DefinedTracker::NotDefinedMacro)
       IfNDefMacro = DT.TheMacro;
 
+    // Restore 'DisableMacroExpansion'.
+    DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
     return ResVal.Val != 0;
   }
 
@@ -712,6 +725,9 @@ EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
     // Parse error, skip the rest of the macro line.
     if (Tok.isNot(tok::eom))
       DiscardUntilEndOfDirective();
+    
+    // Restore 'DisableMacroExpansion'.
+    DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
     return false;
   }
 
@@ -722,6 +738,8 @@ EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro) {
     DiscardUntilEndOfDirective();
   }
 
+  // Restore 'DisableMacroExpansion'.
+  DisableMacroExpansion = DisableMacroExpansionAtStartOfDirective;
   return ResVal.Val != 0;
 }
 
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index 8a61d7b9c247..ce1b19ca7c10 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -64,7 +64,8 @@ PreprocessorLexer *Preprocessor::getCurrentFileLexer() const {
 
 /// EnterSourceFile - Add a source file to the top of the include stack and
 /// start lexing tokens from it instead of the current buffer.
-bool Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir) {
+bool Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir,
+                                   std::string &ErrorStr) {
   assert(CurTokenLexer == 0 && "Cannot #include a file inside a macro!");
   ++NumEnteredSourceFiles;
 
@@ -79,8 +80,9 @@ bool Preprocessor::EnterSourceFile(FileID FID, const DirectoryLookup *CurDir) {
   }
   
   // Get the MemoryBuffer for this FID, if it fails, we fail.
-  const llvm::MemoryBuffer *InputFile = getSourceManager().getBuffer(FID);
-  if (InputFile == 0)
+  const llvm::MemoryBuffer *InputFile =
+    getSourceManager().getBuffer(FID, &ErrorStr);
+  if (!ErrorStr.empty())
     return true;
   
   EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp
index 699b701ea870..dfb14ff06f1f 100644
--- a/lib/Lex/PPMacroExpansion.cpp
+++ b/lib/Lex/PPMacroExpansion.cpp
@@ -204,7 +204,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   // expansion stack, only to take it right back off.
   if (MI->getNumTokens() == 0) {
     // No need for arg info.
-    if (Args) Args->destroy();
+    if (Args) Args->destroy(*this);
 
     // Ignore this macro use, just return the next token in the current
     // buffer.
@@ -232,7 +232,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
     // "#define VAL 42".
 
     // No need for arg info.
-    if (Args) Args->destroy();
+    if (Args) Args->destroy(*this);
 
     // Propagate the isAtStartOfLine/hasLeadingSpace markers of the macro
     // identifier to the expanded token.
@@ -446,7 +446,7 @@ MacroArgs *Preprocessor::ReadFunctionLikeMacroArgs(Token &MacroName,
   }
 
   return MacroArgs::create(MI, ArgTokens.data(), ArgTokens.size(),
-                           isVarargsElided);
+                           isVarargsElided, *this);
 }
 
 /// ComputeDATE_TIME - Compute the current time, enter it into the specified
@@ -486,6 +486,12 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
   case 6:
     if (II->isStr("blocks")) return LangOpts.Blocks;
     return false;
+  case 8:
+    if (II->isStr("cxx_rtti")) return LangOpts.RTTI;
+    return false;
+  case 14:
+    if (II->isStr("cxx_exceptions")) return LangOpts.Exceptions;
+    return false;      
   case 19:
     if (II->isStr("objc_nonfragile_abi")) return LangOpts.ObjCNonFragileABI;
     return false;
@@ -667,7 +673,6 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     // __BASE_FILE__ is a GNU extension that returns the top of the presumed
     // #include stack instead of the current file.
     if (II == Ident__BASE_FILE__) {
-      Diag(Tok, diag::ext_pp_base_file);
       SourceLocation NextLoc = PLoc.getIncludeLoc();
       while (NextLoc.isValid()) {
         PLoc = SourceMgr.getPresumedLoc(NextLoc);
@@ -697,8 +702,6 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
                                                      Tok.getLocation(),
                                                      Tok.getLength()));
   } else if (II == Ident__INCLUDE_LEVEL__) {
-    Diag(Tok, diag::ext_pp_include_level);
-
     // Compute the presumed include depth of this token.  This can be affected
     // by GNU line markers.
     unsigned Depth = 0;
@@ -715,7 +718,6 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
   } else if (II == Ident__TIMESTAMP__) {
     // MSVC, ICC, GCC, VisualAge C++ extension.  The generated string should be
     // of the form "Ddd Mmm dd hh::mm::ss yyyy", which is returned by asctime.
-    Diag(Tok, diag::ext_pp_timestamp);
 
     // Get the file that we are lexing out of.  If we're currently lexing from
     // a macro, dig into the include stack.
@@ -725,7 +727,6 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     if (TheLexer)
       CurFile = SourceMgr.getFileEntryForID(TheLexer->getFileID());
 
-    // If this file is older than the file it depends on, emit a diagnostic.
     const char *Result;
     if (CurFile) {
       time_t TT = CurFile->getModificationTime();
@@ -741,8 +742,6 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     Tok.setKind(tok::string_literal);
     CreateString(TmpBuffer, Len+1, Tok, Tok.getLocation());
   } else if (II == Ident__COUNTER__) {
-    Diag(Tok, diag::ext_pp_counter);
-
     // __COUNTER__ expands to a simple numeric value.
     sprintf(TmpBuffer, "%u", CounterValue++);
     Tok.setKind(tok::numeric_constant);
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 066909475fe3..d4e441b2f183 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -26,6 +26,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Lex/Preprocessor.h"
+#include "MacroArgs.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Pragma.h"
@@ -50,7 +51,8 @@ Preprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts,
                            bool OwnsHeaders)
   : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()),
     SourceMgr(SM), HeaderInfo(Headers), Identifiers(opts, IILookup),
-    BuiltinInfo(Target), CurPPLexer(0), CurDirLookup(0), Callbacks(0) {
+    BuiltinInfo(Target), CodeCompletionFile(0), CurPPLexer(0), CurDirLookup(0),
+    Callbacks(0), MacroArgCache(0) {
   ScratchBuf = new ScratchBuffer(SourceMgr);
   CounterValue = 0; // __COUNTER__ starts at 0.
   OwnsHeaderSearch = OwnsHeaders;
@@ -101,7 +103,7 @@ Preprocessor::~Preprocessor() {
        Macros.begin(), E = Macros.end(); I != E; ++I) {
     // We don't need to free the MacroInfo objects directly.  These
     // will be released when the BumpPtrAllocator 'BP' object gets
-    // destroyed. We still need to run the dstor, however, to free
+    // destroyed.  We still need to run the dtor, however, to free
     // memory alocated by MacroInfo.
     I->second->Destroy(BP);
     I->first->setHasMacroDefinition(false);
@@ -110,6 +112,10 @@ Preprocessor::~Preprocessor() {
   // Free any cached macro expanders.
   for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i)
     delete TokenLexerCache[i];
+  
+  // Free any cached MacroArgs.
+  for (MacroArgs *ArgList = MacroArgCache; ArgList; )
+    ArgList = ArgList->deallocate();
 
   // Release pragma information.
   delete PragmaHandlers;
@@ -188,6 +194,57 @@ void Preprocessor::PrintStats() {
              << NumFastTokenPaste << " on the fast path.\n";
 }
 
+bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, 
+                                          unsigned TruncateAtLine, 
+                                          unsigned TruncateAtColumn) {
+  using llvm::MemoryBuffer;
+
+  CodeCompletionFile = File;
+
+  // Okay to clear out the code-completion point by passing NULL.
+  if (!CodeCompletionFile)
+    return false;
+
+  // Load the actual file's contents.
+  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File);
+  if (!Buffer)
+    return true;
+
+  // Find the byte position of the truncation point.
+  const char *Position = Buffer->getBufferStart();
+  for (unsigned Line = 1; Line < TruncateAtLine; ++Line) {
+    for (; *Position; ++Position) {
+      if (*Position != '\r' && *Position != '\n')
+        continue;
+      
+      // Eat \r\n or \n\r as a single line.
+      if ((Position[1] == '\r' || Position[1] == '\n') &&
+          Position[0] != Position[1])
+        ++Position;
+      ++Position;
+      break;
+    }
+  }
+  
+  Position += TruncateAtColumn - 1;
+  
+  // Truncate the buffer.
+  if (Position < Buffer->getBufferEnd()) {
+    MemoryBuffer *TruncatedBuffer 
+      = MemoryBuffer::getMemBufferCopy(Buffer->getBufferStart(), Position, 
+                                       Buffer->getBufferIdentifier());
+    SourceMgr.overrideFileContents(File, TruncatedBuffer);
+  }
+
+  return false;
+}
+
+bool Preprocessor::isCodeCompletionFile(SourceLocation FileLoc) const {
+  return CodeCompletionFile && FileLoc.isFileID() &&
+    SourceMgr.getFileEntryForID(SourceMgr.getFileID(FileLoc))
+      == CodeCompletionFile;
+}
+
 //===----------------------------------------------------------------------===//
 // Token Spelling
 //===----------------------------------------------------------------------===//
@@ -380,7 +437,9 @@ void Preprocessor::EnterMainSourceFile() {
   FileID MainFileID = SourceMgr.getMainFileID();
 
   // Enter the main file source buffer.
-  EnterSourceFile(MainFileID, 0);
+  std::string ErrorStr;
+  bool Res = EnterSourceFile(MainFileID, 0, ErrorStr);
+  assert(!Res && "Entering main file should not fail!");
 
   // Tell the header info that the main file was entered.  If the file is later
   // #imported, it won't be re-entered.
@@ -406,7 +465,8 @@ void Preprocessor::EnterMainSourceFile() {
   assert(!FID.isInvalid() && "Could not create FileID for predefines?");
 
   // Start parsing the predefines.
-  EnterSourceFile(FID, 0);
+  Res = EnterSourceFile(FID, 0, ErrorStr);
+  assert(!Res && "Entering predefines should not fail!");
 }
 
 
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index f006f5ae55bb..a40bb62db46d 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -92,7 +92,7 @@ void TokenLexer::destroy() {
   }
 
   // TokenLexer owns its formal arguments.
-  if (ActualArgs) ActualArgs->destroy();
+  if (ActualArgs) ActualArgs->destroy(PP);
 }
 
 /// Expand the arguments of a function-like macro so that we can quickly
@@ -321,13 +321,12 @@ void TokenLexer::Lex(Token &Tok) {
 
   // If this token is followed by a token paste (##) operator, paste the tokens!
   if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)) {
-    if (PasteTokens(Tok)) {
-      // When handling the microsoft /##/ extension, the final token is
-      // returned by PasteTokens, not the pasted token.
+    // When handling the microsoft /##/ extension, the final token is
+    // returned by PasteTokens, not the pasted token.
+    if (PasteTokens(Tok))
       return;
-    } else {
-      TokenIsFromPaste = true;
-    }
+    
+    TokenIsFromPaste = true;
   }
 
   // The token's current location indicate where the token was lexed from.  We