vendor/clang/clang-r142614

19 files changed, 799 insertions, 266 deletions

diff --git a/include/clang/Lex/CodeCompletionHandler.h b/include/clang/Lex/CodeCompletionHandler.h
index d28a3aa7d6309..d876776c927c1 100644
--- a/include/clang/Lex/CodeCompletionHandler.h
+++ b/include/clang/Lex/CodeCompletionHandler.h
@@ -52,6 +52,10 @@ public:
   
   /// \brief Callback invoked when performing code completion inside a 
   /// function-like macro argument.
+  ///
+  /// There will be another callback invocation after the macro arguments are
+  /// parsed, so this callback should generally be used to note that the next
+  /// callback is invoked inside a macro argument.
   virtual void CodeCompleteMacroArgument(IdentifierInfo *Macro,
                                          MacroInfo *MacroInfo,
                                          unsigned ArgumentIndex) { }
diff --git a/include/clang/Lex/DirectoryLookup.h b/include/clang/Lex/DirectoryLookup.h
index 1ee6953a12b2b..f7da61b0ed710 100644
--- a/include/clang/Lex/DirectoryLookup.h
+++ b/include/clang/Lex/DirectoryLookup.h
@@ -14,12 +14,9 @@
 #ifndef LLVM_CLANG_LEX_DIRECTORYLOOKUP_H
 #define LLVM_CLANG_LEX_DIRECTORYLOOKUP_H
 
+#include "clang/Basic/LLVM.h"
 #include "clang/Basic/SourceManager.h"
 
-namespace llvm {
-  class StringRef;
-  template <typename T> class SmallVectorImpl;
-}
 namespace clang {
 class HeaderMap;
 class DirectoryEntry;
@@ -59,21 +56,27 @@ private:
   /// LookupType - This indicates whether this DirectoryLookup object is a
   /// normal directory, a framework, or a headermap.
   unsigned LookupType : 2;
+  
+  /// \brief Whether this is a header map used when building a framework.
+  unsigned IsIndexHeaderMap : 1;
+  
 public:
   /// DirectoryLookup ctor - Note that this ctor *does not take ownership* of
   /// 'dir'.
   DirectoryLookup(const DirectoryEntry *dir, SrcMgr::CharacteristicKind DT,
                   bool isUser, bool isFramework)
-    : DirCharacteristic(DT), UserSupplied(isUser),
-     LookupType(isFramework ? LT_Framework : LT_NormalDir) {
+    : DirCharacteristic(DT), UserSupplied(isUser), 
+      LookupType(isFramework ? LT_Framework : LT_NormalDir),
+      IsIndexHeaderMap(false) {
     u.Dir = dir;
   }
 
   /// DirectoryLookup ctor - Note that this ctor *does not take ownership* of
   /// 'map'.
   DirectoryLookup(const HeaderMap *map, SrcMgr::CharacteristicKind DT,
-                  bool isUser)
-    : DirCharacteristic(DT), UserSupplied(isUser), LookupType(LT_HeaderMap) {
+                  bool isUser, bool isIndexHeaderMap)
+    : DirCharacteristic(DT), UserSupplied(isUser), LookupType(LT_HeaderMap),
+      IsIndexHeaderMap(isIndexHeaderMap) {
     u.Map = map;
   }
 
@@ -119,7 +122,11 @@ public:
   ///
   bool isUserSupplied() const { return UserSupplied; }
 
-
+  /// \brief Whether this header map is building a framework or not.
+  bool isIndexHeaderMap() const { 
+    return isHeaderMap() && IsIndexHeaderMap; 
+  }
+  
   /// LookupFile - Lookup the specified file in this search path, returning it
   /// if it exists or returning null if not.
   ///
@@ -133,15 +140,25 @@ public:
   /// \param RelativePath If not NULL, will be set to the path relative to
   /// SearchPath at which the file was found. This only differs from the
   /// Filename for framework includes.
-  const FileEntry *LookupFile(llvm::StringRef Filename, HeaderSearch &HS,
-                              llvm::SmallVectorImpl<char> *SearchPath,
-                              llvm::SmallVectorImpl<char> *RelativePath) const;
+  ///
+  /// \param BuildingModule The name of the module we're currently building.
+  ///
+  /// \param SuggestedModule If non-null, and the file found is semantically
+  /// part of a known module, this will be set to the name of the module that
+  /// could be imported instead of preprocessing/parsing the file found.
+  const FileEntry *LookupFile(StringRef Filename, HeaderSearch &HS,
+                              SmallVectorImpl<char> *SearchPath,
+                              SmallVectorImpl<char> *RelativePath,
+                              StringRef BuildingModule,
+                              StringRef *SuggestedModule) const;
 
 private:
   const FileEntry *DoFrameworkLookup(
-      llvm::StringRef Filename, HeaderSearch &HS,
-      llvm::SmallVectorImpl<char> *SearchPath,
-      llvm::SmallVectorImpl<char> *RelativePath) const;
+      StringRef Filename, HeaderSearch &HS,
+      SmallVectorImpl<char> *SearchPath,
+      SmallVectorImpl<char> *RelativePath,
+      StringRef BuildingModule,
+      StringRef *SuggestedModule) const;
 
 };
 
diff --git a/include/clang/Lex/HeaderMap.h b/include/clang/Lex/HeaderMap.h
index e333840b6a9db..08bc5b64bc129 100644
--- a/include/clang/Lex/HeaderMap.h
+++ b/include/clang/Lex/HeaderMap.h
@@ -14,10 +14,10 @@
 #ifndef LLVM_CLANG_LEX_HEADERMAP_H
 #define LLVM_CLANG_LEX_HEADERMAP_H
 
+#include "clang/Basic/LLVM.h"
+
 namespace llvm {
   class MemoryBuffer;
-  class StringRef;
-  template <typename T> class SmallVectorImpl;
 }
 namespace clang {
   class FileEntry;
@@ -52,7 +52,7 @@ public:
   /// raw path at which the file was found in the file system. For example,
   /// for a search path ".." and a filename "../file.h" this would be
   /// "../../file.h".
-  const FileEntry *LookupFile(llvm::StringRef Filename, FileManager &FM) const;
+  const FileEntry *LookupFile(StringRef Filename, FileManager &FM) const;
 
   /// getFileName - Return the filename of the headermap.
   const char *getFileName() const;
diff --git a/include/clang/Lex/HeaderSearch.h b/include/clang/Lex/HeaderSearch.h
index 5e36d8e608213..84d59f793bbb8 100644
--- a/include/clang/Lex/HeaderSearch.h
+++ b/include/clang/Lex/HeaderSearch.h
@@ -16,6 +16,8 @@
 
 #include "clang/Lex/DirectoryLookup.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Allocator.h"
 #include <vector>
 
 namespace clang {
@@ -47,6 +49,15 @@ struct HeaderFileInfo {
   /// "resolved", meaning that it was loaded from the external source.
   unsigned Resolved : 1;
   
+  /// \brief Whether this is a header inside a framework that is currently
+  /// being built. 
+  ///
+  /// When a framework is being built, the headers have not yet been placed
+  /// into the appropriate framework subdirectories, and therefore are
+  /// provided via a header map. This bit indicates when this is one of
+  /// those framework headers.
+  unsigned IndexHeaderMapHeader : 1;
+  
   /// NumIncludes - This is the number of times the file has been included
   /// already.
   unsigned short NumIncludes;
@@ -68,10 +79,14 @@ struct HeaderFileInfo {
   /// external storage.
   const IdentifierInfo *ControllingMacro;
 
+  /// \brief If this header came from a framework include, this is the name
+  /// of the framework.
+  StringRef Framework;
+  
   HeaderFileInfo()
     : isImport(false), isPragmaOnce(false), DirInfo(SrcMgr::C_User), 
-      External(false), Resolved(false), NumIncludes(0), ControllingMacroID(0), 
-      ControllingMacro(0)  {}
+      External(false), Resolved(false), IndexHeaderMapHeader(false),
+      NumIncludes(0), ControllingMacroID(0), ControllingMacro(0)  {}
 
   /// \brief Retrieve the controlling macro for this header file, if
   /// any.
@@ -114,6 +129,12 @@ class HeaderSearch {
   unsigned SystemDirIdx;
   bool NoCurDirSearch;
 
+  /// \brief The path to the module cache.
+  std::string ModuleCachePath;
+  
+  /// \brief The name of the module we're building.
+  std::string BuildingModule;
+  
   /// FileInfo - This contains all of the preprocessor-specific data about files
   /// that are included.  The vector is indexed by the FileEntry's UID.
   ///
@@ -125,17 +146,23 @@ class HeaderSearch {
   /// and this value doesn't match the current query, the cache has to be
   /// ignored.  The second value is the entry in SearchDirs that satisfied the
   /// query.
-  llvm::StringMap<std::pair<unsigned, unsigned> > LookupFileCache;
+  llvm::StringMap<std::pair<unsigned, unsigned>, llvm::BumpPtrAllocator>
+    LookupFileCache;
 
 
   /// FrameworkMap - This is a collection mapping a framework or subframework
   /// name like "Carbon" to the Carbon.framework directory.
-  llvm::StringMap<const DirectoryEntry *> FrameworkMap;
+  llvm::StringMap<const DirectoryEntry *, llvm::BumpPtrAllocator>
+    FrameworkMap;
 
   /// HeaderMaps - This is a mapping from FileEntry -> HeaderMap, uniquing
   /// headermaps.  This vector owns the headermap.
   std::vector<std::pair<const FileEntry*, const HeaderMap*> > HeaderMaps;
 
+  /// \brief Uniqued set of framework names, which is used to track which 
+  /// headers were included as framework headers.
+  llvm::StringSet<llvm::BumpPtrAllocator> FrameworkNames;
+  
   /// \brief Entity used to resolve the identifier IDs of controlling
   /// macros into IdentifierInfo pointers, as needed.
   ExternalIdentifierLookup *ExternalLookup;
@@ -172,6 +199,13 @@ public:
     //LookupFileCache.clear();
   }
 
+  /// \brief Set the path to the module cache and the name of the module
+  /// we're building
+  void configureModules(StringRef CachePath, StringRef BuildingModule) {
+    ModuleCachePath = CachePath;
+    this->BuildingModule = BuildingModule;
+  }
+  
   /// ClearFileInfo - Forget everything we know about headers so far.
   void ClearFileInfo() {
     FileInfo.clear();
@@ -211,12 +245,17 @@ public:
   /// \param RelativePath If non-null, will be set to the path relative to
   /// SearchPath at which the file was found. This only differs from the
   /// Filename for framework includes.
-  const FileEntry *LookupFile(llvm::StringRef Filename, bool isAngled,
+  ///
+  /// \param SuggestedModule If non-null, and the file found is semantically
+  /// part of a known module, this will be set to the name of the module that
+  /// could be imported instead of preprocessing/parsing the file found.
+  const FileEntry *LookupFile(StringRef Filename, bool isAngled,
                               const DirectoryLookup *FromDir,
                               const DirectoryLookup *&CurDir,
                               const FileEntry *CurFileEnt,
-                              llvm::SmallVectorImpl<char> *SearchPath,
-                              llvm::SmallVectorImpl<char> *RelativePath);
+                              SmallVectorImpl<char> *SearchPath,
+                              SmallVectorImpl<char> *RelativePath,
+                              StringRef *SuggestedModule);
 
   /// LookupSubframeworkHeader - Look up a subframework for the specified
   /// #include file.  For example, if #include'ing <HIToolbox/HIToolbox.h> from
@@ -224,15 +263,15 @@ public:
   /// is a subframework within Carbon.framework.  If so, return the FileEntry
   /// for the designated file, otherwise return null.
   const FileEntry *LookupSubframeworkHeader(
-      llvm::StringRef Filename,
+      StringRef Filename,
       const FileEntry *RelativeFileEnt,
-      llvm::SmallVectorImpl<char> *SearchPath,
-      llvm::SmallVectorImpl<char> *RelativePath);
+      SmallVectorImpl<char> *SearchPath,
+      SmallVectorImpl<char> *RelativePath);
 
   /// LookupFrameworkCache - Look up the specified framework name in our
   /// framework cache, returning the DirectoryEntry it is in if we know,
   /// otherwise, return null.
-  const DirectoryEntry *&LookupFrameworkCache(llvm::StringRef FWName) {
+  const DirectoryEntry *&LookupFrameworkCache(StringRef FWName) {
     return FrameworkMap.GetOrCreateValue(FWName).getValue();
   }
 
@@ -287,6 +326,23 @@ public:
   /// FileEntry, uniquing them through the the 'HeaderMaps' datastructure.
   const HeaderMap *CreateHeaderMap(const FileEntry *FE);
 
+  /// \brief Search in the module cache path for a module with the given
+  /// name.
+  ///
+  /// \param If non-NULL, will be set to the module file name we expected to
+  /// find (regardless of whether it was actually found or not).
+  ///
+  /// \param UmbrellaHeader If non-NULL, and no module was found in the module
+  /// cache, this routine will search in the framework paths to determine
+  /// whether a module can be built from an umbrella header. If so, the pointee
+  /// will be set to the path of the umbrella header.
+  ///
+  /// \returns A file describing the named module, if available, or NULL to
+  /// indicate that the module could not be found.
+  const FileEntry *lookupModule(StringRef ModuleName,
+                                std::string *ModuleFileName = 0,
+                                std::string *UmbrellaHeader = 0);
+  
   void IncrementFrameworkLookupCount() { ++NumFrameworkLookups; }
 
   typedef std::vector<HeaderFileInfo>::const_iterator header_file_iterator;
@@ -322,7 +378,13 @@ public:
   }
   search_dir_iterator system_dir_end() const { return SearchDirs.end(); }
 
+  /// \brief Retrieve a uniqued framework name.
+  StringRef getUniqueFrameworkName(StringRef Framework);
+  
   void PrintStats();
+  
+  size_t getTotalMemory() const;
+
 private:
 
   /// getFileInfo - Return the HeaderFileInfo structure for the specified
diff --git a/include/clang/Lex/LexDiagnostic.h b/include/clang/Lex/LexDiagnostic.h
index 7d2eb89c50bc2..f454e2309acbf 100644
--- a/include/clang/Lex/LexDiagnostic.h
+++ b/include/clang/Lex/LexDiagnostic.h
@@ -16,7 +16,7 @@ namespace clang {
   namespace diag {
     enum {
 #define DIAG(ENUM,FLAGS,DEFAULT_MAPPING,DESC,GROUP,\
-             SFINAE,ACCESS,CATEGORY,BRIEF,FULL) ENUM,
+             SFINAE,ACCESS,NOWERROR,SHOWINSYSHEADER,CATEGORY,BRIEF,FULL) ENUM,
 #define LEXSTART
 #include "clang/Basic/DiagnosticLexKinds.inc"
 #undef DIAG
diff --git a/include/clang/Lex/Lexer.h b/include/clang/Lex/Lexer.h
index f4297627e86fe..e01427f574d28 100644
--- a/include/clang/Lex/Lexer.h
+++ b/include/clang/Lex/Lexer.h
@@ -21,11 +21,24 @@
 #include <cassert>
 
 namespace clang {
-class Diagnostic;
+class DiagnosticsEngine;
 class SourceManager;
 class Preprocessor;
 class DiagnosticBuilder;
 
+/// ConflictMarkerKind - Kinds of conflict marker which the lexer might be
+/// recovering from.
+enum ConflictMarkerKind {
+  /// Not within a conflict marker.
+  CMK_None,
+  /// A normal or diff3 conflict marker, initiated by at least 7 <s,
+  /// separated by at least 7 =s or |s, and terminated by at least 7 >s.
+  CMK_Normal,
+  /// A Perforce-style conflict marker, initiated by 4 >s, separated by 4 =s,
+  /// and terminated by 4 <s.
+  CMK_Perforce
+};
+
 /// Lexer - This provides a simple interface that turns a text buffer into a
 /// stream of tokens.  This provides no support for file reading or buffering,
 /// or buffering/seeking of tokens, only forward lexing is supported.  It relies
@@ -37,8 +50,7 @@ class Lexer : public PreprocessorLexer {
   const char *BufferEnd;         // End of the buffer.
   SourceLocation FileLoc;        // Location for start of file.
   LangOptions Features;          // Features enabled by this language (cache).
-  bool Is_PragmaLexer : 1;       // True if lexer for _Pragma handling.
-  bool IsInConflictMarker : 1;   // True if in a VCS conflict marker '<<<<<<<'
+  bool Is_PragmaLexer;           // True if lexer for _Pragma handling.
   
   //===--------------------------------------------------------------------===//
   // Context-specific lexing flags set by the preprocessor.
@@ -66,6 +78,9 @@ class Lexer : public PreprocessorLexer {
   // line" flag set on it.
   bool IsAtStartOfLine;
 
+  // CurrentConflictMarkerState - The kind of conflict marker we are handling.
+  ConflictMarkerKind CurrentConflictMarkerState;
+
   Lexer(const Lexer&);          // DO NOT IMPLEMENT
   void operator=(const Lexer&); // DO NOT IMPLEMENT
   friend class Preprocessor;
@@ -208,7 +223,7 @@ public:
 
   /// Stringify - Convert the specified string into a C string by escaping '\'
   /// and " characters.  This does not add surrounding ""'s to the string.
-  static void Stringify(llvm::SmallVectorImpl<char> &Str);
+  static void Stringify(SmallVectorImpl<char> &Str);
 
   
   /// getSpelling - This method is used to get the spelling of a token into a
@@ -244,8 +259,8 @@ public:
   /// This method lexes at the expansion depth of the given
   /// location and does not jump to the expansion or spelling
   /// location.
-  static llvm::StringRef getSpelling(SourceLocation loc,
-                                     llvm::SmallVectorImpl<char> &buffer,
+  static StringRef getSpelling(SourceLocation loc,
+                                     SmallVectorImpl<char> &buffer,
                                      const SourceManager &SourceMgr,
                                      const LangOptions &Features,
                                      bool *invalid = 0);
@@ -322,7 +337,8 @@ public:
   /// of the file begins along with a boolean value indicating whether 
   /// the preamble ends at the beginning of a new line.
   static std::pair<unsigned, bool>
-  ComputePreamble(const llvm::MemoryBuffer *Buffer, unsigned MaxLines = 0);
+  ComputePreamble(const llvm::MemoryBuffer *Buffer, const LangOptions &Features,
+                  unsigned MaxLines = 0);
                                         
   //===--------------------------------------------------------------------===//
   // Internal implementation interfaces.
@@ -456,6 +472,18 @@ public:
   /// them), skip over them and return the first non-escaped-newline found,
   /// otherwise return P.
   static const char *SkipEscapedNewLines(const char *P);
+
+  /// \brief Checks that the given token is the first token that occurs after
+  /// the given location (this excludes comments and whitespace). Returns the
+  /// location immediately after the specified token. If the token is not found
+  /// or the location is inside a macro, the returned source location will be
+  /// invalid.
+  static SourceLocation findLocationAfterToken(SourceLocation loc,
+                                         tok::TokenKind TKind,
+                                         const SourceManager &SM,
+                                         const LangOptions &LangOpts,
+                                         bool SkipTrailingWhitespaceAndNewLine);
+
 private:
 
   /// getCharAndSizeSlowNoWarn - Same as getCharAndSizeSlow, but never emits a
@@ -471,9 +499,13 @@ private:
   // Helper functions to lex the remainder of a token of the specific type.
   void LexIdentifier         (Token &Result, const char *CurPtr);
   void LexNumericConstant    (Token &Result, const char *CurPtr);
-  void LexStringLiteral      (Token &Result, const char *CurPtr,bool Wide);
+  void LexStringLiteral      (Token &Result, const char *CurPtr,
+                              tok::TokenKind Kind);
+  void LexRawStringLiteral   (Token &Result, const char *CurPtr,
+                              tok::TokenKind Kind);
   void LexAngledStringLiteral(Token &Result, const char *CurPtr);
-  void LexCharConstant       (Token &Result, const char *CurPtr);
+  void LexCharConstant       (Token &Result, const char *CurPtr,
+                              tok::TokenKind Kind);
   bool LexEndOfFile          (Token &Result, const char *CurPtr);
 
   bool SkipWhitespace        (Token &Result, const char *CurPtr);
@@ -483,6 +515,9 @@ private:
   
   bool IsStartOfConflictMarker(const char *CurPtr);
   bool HandleEndOfConflictMarker(const char *CurPtr);
+
+  bool isCodeCompletionPoint(const char *CurPtr) const;
+  void cutOffLexing() { BufferPtr = BufferEnd; }
 };
 
 
diff --git a/include/clang/Lex/LiteralSupport.h b/include/clang/Lex/LiteralSupport.h
index 0dbcd6d72d63d..b33092c753a8c 100644
--- a/include/clang/Lex/LiteralSupport.h
+++ b/include/clang/Lex/LiteralSupport.h
@@ -15,14 +15,16 @@
 #ifndef CLANG_LITERALSUPPORT_H
 #define CLANG_LITERALSUPPORT_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/DataTypes.h"
+#include "clang/Basic/TokenKinds.h"
 #include <cctype>
 
 namespace clang {
 
-class Diagnostic;
+class DiagnosticsEngine;
 class Preprocessor;
 class Token;
 class SourceLocation;
@@ -123,15 +125,19 @@ private:
 /// character literal.
 class CharLiteralParser {
   uint64_t Value;
-  bool IsWide;
+  tok::TokenKind Kind;
   bool IsMultiChar;
   bool HadError;
 public:
   CharLiteralParser(const char *begin, const char *end,
-                    SourceLocation Loc, Preprocessor &PP);
+                    SourceLocation Loc, Preprocessor &PP,
+                    tok::TokenKind kind);
 
   bool hadError() const { return HadError; }
-  bool isWide() const { return IsWide; }
+  bool isAscii() const { return Kind == tok::char_constant; }
+  bool isWide() const { return Kind == tok::wide_char_constant; }
+  bool isUTF16() const { return Kind == tok::utf16_char_constant; }
+  bool isUTF32() const { return Kind == tok::utf32_char_constant; }
   bool isMultiChar() const { return IsMultiChar; }
   uint64_t getValue() const { return Value; }
 };
@@ -143,11 +149,12 @@ class StringLiteralParser {
   const SourceManager &SM;
   const LangOptions &Features;
   const TargetInfo &Target;
-  Diagnostic *Diags;
+  DiagnosticsEngine *Diags;
   
   unsigned MaxTokenLength;
   unsigned SizeBound;
-  unsigned wchar_tByteWidth;
+  unsigned CharByteWidth;
+  tok::TokenKind Kind;
   llvm::SmallString<512> ResultBuf;
   char *ResultPtr; // cursor
 public:
@@ -155,27 +162,24 @@ public:
                       Preprocessor &PP, bool Complain = true);
   StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
                       const SourceManager &sm, const LangOptions &features,
-                      const TargetInfo &target, Diagnostic *diags = 0)
+                      const TargetInfo &target, DiagnosticsEngine *diags = 0)
     : SM(sm), Features(features), Target(target), Diags(diags),
-      MaxTokenLength(0), SizeBound(0), wchar_tByteWidth(0),
-      ResultPtr(ResultBuf.data()), hadError(false), AnyWide(false), Pascal(false) {
+      MaxTokenLength(0), SizeBound(0), CharByteWidth(0), Kind(tok::unknown),
+      ResultPtr(ResultBuf.data()), hadError(false), Pascal(false) {
     init(StringToks, NumStringToks);
   }
     
 
   bool hadError;
-  bool AnyWide;
   bool Pascal;
 
-  llvm::StringRef GetString() const {
-    return llvm::StringRef(ResultBuf.data(), GetStringLength());
+  StringRef GetString() const {
+    return StringRef(ResultBuf.data(), GetStringLength());
   }
   unsigned GetStringLength() const { return ResultPtr-ResultBuf.data(); }
 
   unsigned GetNumStringChars() const {
-    if (AnyWide)
-      return GetStringLength() / wchar_tByteWidth;
-    return GetStringLength();
+    return GetStringLength() / CharByteWidth;
   }
   /// getOffsetOfStringByte - This function returns the offset of the
   /// specified byte of the string data represented by Token.  This handles
@@ -184,9 +188,16 @@ public:
   /// If the Diagnostics pointer is non-null, then this will do semantic
   /// checking of the string literal and emit errors and warnings.
   unsigned getOffsetOfStringByte(const Token &TheTok, unsigned ByteNo) const;
-  
+
+  bool isAscii() { return Kind == tok::string_literal; }
+  bool isWide() { return Kind == tok::wide_string_literal; }
+  bool isUTF8() { return Kind == tok::utf8_string_literal; }
+  bool isUTF16() { return Kind == tok::utf16_string_literal; }
+  bool isUTF32() { return Kind == tok::utf32_string_literal; }
+
 private:
   void init(const Token *StringToks, unsigned NumStringToks);
+  void CopyStringFragment(StringRef Fragment);
 };
 
 }  // end namespace clang
diff --git a/include/clang/Lex/MacroInfo.h b/include/clang/Lex/MacroInfo.h
index 9e9d7cf500a42..b381e0f25f41a 100644
--- a/include/clang/Lex/MacroInfo.h
+++ b/include/clang/Lex/MacroInfo.h
@@ -39,9 +39,14 @@ class MacroInfo {
   IdentifierInfo **ArgumentList;
   unsigned NumArguments;
 
+  /// \brief The location at which this macro was exported from its module.
+  ///
+  /// If invalid, this macro has not been explicitly exported.
+  SourceLocation ExportLocation;
+  
   /// ReplacementTokens - This is the list of tokens that the macro is defined
   /// to.
-  llvm::SmallVector<Token, 8> ReplacementTokens;
+  SmallVector<Token, 8> ReplacementTokens;
 
   /// \brief Length in characters of the macro definition.
   mutable unsigned DefinitionLength;
@@ -68,6 +73,9 @@ class MacroInfo {
   /// IsFromAST - True if this macro was loaded from an AST file.
   bool IsFromAST : 1;
 
+  /// \brief Whether this macro changed after it was loaded from an AST file.
+  bool ChangedAfterLoad : 1;
+  
 private:
   //===--------------------------------------------------------------------===//
   // State that changes as the macro is used.
@@ -209,6 +217,14 @@ public:
   /// setIsFromAST - Set whether this macro was loaded from an AST file.
   void setIsFromAST(bool FromAST = true) { IsFromAST = FromAST; }
 
+  /// \brief Determine whether this macro has changed since it was loaded from
+  /// an AST file.
+  bool hasChangedAfterLoad() const { return ChangedAfterLoad; }
+  
+  /// \brief Note whether this macro has changed after it was loaded from an
+  /// AST file.
+  void setChangedAfterLoad(bool CAL = true) { ChangedAfterLoad = CAL; }
+  
   /// isUsed - Return false if this macro is defined in the main file and has
   /// not yet been used.
   bool isUsed() const { return IsUsed; }
@@ -235,7 +251,7 @@ public:
     return ReplacementTokens[Tok];
   }
 
-  typedef llvm::SmallVector<Token, 8>::const_iterator tokens_iterator;
+  typedef SmallVector<Token, 8>::const_iterator tokens_iterator;
   tokens_iterator tokens_begin() const { return ReplacementTokens.begin(); }
   tokens_iterator tokens_end() const { return ReplacementTokens.end(); }
   bool tokens_empty() const { return ReplacementTokens.empty(); }
@@ -262,6 +278,19 @@ public:
     IsDisabled = true;
   }
 
+  /// \brief Set the export location for this macro.
+  void setExportLocation(SourceLocation ExportLoc) {
+    ExportLocation = ExportLoc;
+  }
+
+  /// \brief Determine whether this macro was explicitly exported from its
+  /// module.
+  bool isExported() const { return ExportLocation.isValid(); }
+  
+  /// \brief Determine the location where this macro was explicitly exported
+  /// from its module.
+  SourceLocation getExportLocation() { return ExportLocation; }
+  
 private:
   unsigned getDefinitionLengthSlow(SourceManager &SM) const;
 };
diff --git a/include/clang/Lex/Makefile b/include/clang/Lex/Makefile
index 9874bcffb3e1d..762b9a2587582 100644
--- a/include/clang/Lex/Makefile
+++ b/include/clang/Lex/Makefile
@@ -6,8 +6,8 @@ TABLEGEN_INC_FILES_COMMON = 1
 
 include $(CLANG_LEVEL)/Makefile
 
-$(ObjDir)/AttrSpellings.inc.tmp : $(TD_SRC_DIR)/Attr.td $(TBLGEN) \
+$(ObjDir)/AttrSpellings.inc.tmp : $(TD_SRC_DIR)/Attr.td $(CLANG_TBLGEN) \
                                   $(ObjDir)/.dir
 	$(Echo) "Building Clang attribute spellings with tblgen"
-	$(Verb) $(TableGen) -gen-clang-attr-spelling-list -o $(call SYSPATH, $@) \
+	$(Verb) $(ClangTableGen) -gen-clang-attr-spelling-list -o $(call SYSPATH, $@) \
 		-I $(PROJ_SRC_DIR)/../../ $<
diff --git a/include/clang/Lex/ModuleLoader.h b/include/clang/Lex/ModuleLoader.h
new file mode 100644
index 0000000000000..72ec0e3ebc9f5
--- /dev/null
+++ b/include/clang/Lex/ModuleLoader.h
@@ -0,0 +1,55 @@
+//===--- ModuleLoader.h - Module Loader Interface ---------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ModuleLoader interface, which is responsible for 
+//  loading named modules.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_LEX_MODULE_LOADER_H
+#define LLVM_CLANG_LEX_MODULE_LOADER_H
+
+#include "clang/Basic/SourceLocation.h"
+
+namespace clang {
+
+class IdentifierInfo;
+  
+/// \brief An opaque key that is used to describe the module and can be 
+/// interpreted by the module loader itself.
+typedef void *ModuleKey;
+  
+/// \brief Abstract interface for a module loader.
+///
+/// This abstract interface describes a module loader, which is responsible
+/// for resolving a module name (e.g., "std") to an actual module file, and
+/// then loading that module.
+class ModuleLoader {
+public:
+  virtual ~ModuleLoader();
+  
+  /// \brief Attempt to load the given module.
+  ///
+  /// This routine attempts to load the module described by the given 
+  /// parameters.
+  ///
+  /// \param ImportLoc The location of the 'import' keyword.
+  /// \param ModuleName The name of the module to be loaded.
+  /// \param ModuleNameLoc The location of the module name.
+  ///
+  /// \returns If successful, a non-NULL module key describing this module.
+  /// Otherwise, returns NULL to indicate that the module could not be
+  /// loaded.
+  virtual ModuleKey loadModule(SourceLocation ImportLoc, 
+                               IdentifierInfo &ModuleName,
+                               SourceLocation ModuleNameLoc) = 0;
+};
+  
+}
+
+#endif
diff --git a/include/clang/Lex/PPCallbacks.h b/include/clang/Lex/PPCallbacks.h
index a7948153a7280..1fc1a05db6462 100644
--- a/include/clang/Lex/PPCallbacks.h
+++ b/include/clang/Lex/PPCallbacks.h
@@ -42,8 +42,11 @@ public:
   /// EnteringFile indicates whether this is because we are entering a new
   /// #include'd file (when true) or whether we're exiting one because we ran
   /// off the end (when false).
+  ///
+  /// \param PrevFID the file that was exited if \arg Reason is ExitFile. 
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType) {
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID = FileID()) {
   }
 
   /// FileSkipped - This callback is invoked whenever a source file is
@@ -90,12 +93,12 @@ public:
   /// file was found. This is equal to FileName except for framework includes.
   virtual void InclusionDirective(SourceLocation HashLoc,
                                   const Token &IncludeTok,
-                                  llvm::StringRef FileName,
+                                  StringRef FileName,
                                   bool IsAngled,
                                   const FileEntry *File,
                                   SourceLocation EndLoc,
-                                  llvm::StringRef SearchPath,
-                                  llvm::StringRef RelativePath) {
+                                  StringRef SearchPath,
+                                  StringRef RelativePath) {
   }
 
   /// EndOfMainFile - This callback is invoked when the end of the main file is
@@ -122,31 +125,32 @@ public:
   /// \param Loc The location of the message directive.
   /// \param str The text of the message directive.
   ///
-  virtual void PragmaMessage(SourceLocation Loc, llvm::StringRef Str) {
+  virtual void PragmaMessage(SourceLocation Loc, StringRef Str) {
   }
 
   /// PragmaDiagnosticPush - This callback is invoked when a
   /// #pragma gcc dianostic push directive is read.
   virtual void PragmaDiagnosticPush(SourceLocation Loc,
-                                    llvm::StringRef Namespace) {
+                                    StringRef Namespace) {
   }
 
   /// PragmaDiagnosticPop - This callback is invoked when a
   /// #pragma gcc dianostic pop directive is read.
   virtual void PragmaDiagnosticPop(SourceLocation Loc,
-                                   llvm::StringRef Namespace) {
+                                   StringRef Namespace) {
   }
 
   /// PragmaDiagnostic - This callback is invoked when a
   /// #pragma gcc dianostic directive is read.
-  virtual void PragmaDiagnostic(SourceLocation Loc, llvm::StringRef Namespace,
-                                diag::Mapping mapping, llvm::StringRef Str) {
+  virtual void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
+                                diag::Mapping mapping, StringRef Str) {
   }
 
   /// MacroExpands - This is called by
   /// Preprocessor::HandleMacroExpandedIdentifier when a macro invocation is
   /// found.
-  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI) {
+  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI,
+                            SourceRange Range) {
   }
 
   /// MacroDefined - This hook is called whenever a macro definition is seen.
@@ -157,6 +161,16 @@ public:
   /// MI is released immediately following this callback.
   virtual void MacroUndefined(const Token &MacroNameTok, const MacroInfo *MI) {
   }
+  
+  /// Defined - This hook is called whenever the 'defined' operator is seen.
+  virtual void Defined(const Token &MacroNameTok) {
+  }
+  
+  /// SourceRangeSkipped - This hook is called when a source range is skipped.
+  /// \param Range The SourceRange that was skipped. The range begins at the
+  /// #if/#else directive and ends after the #endif/#else directive.
+  virtual void SourceRangeSkipped(SourceRange Range) {
+  }
 
   /// If -- This hook is called whenever an #if is seen.
   /// \param Range The SourceRange of the expression being tested.
@@ -204,9 +218,10 @@ public:
   }
 
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType) {
-    First->FileChanged(Loc, Reason, FileType);
-    Second->FileChanged(Loc, Reason, FileType);
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID) {
+    First->FileChanged(Loc, Reason, FileType, PrevFID);
+    Second->FileChanged(Loc, Reason, FileType, PrevFID);
   }
 
   virtual void FileSkipped(const FileEntry &ParentFile,
@@ -218,12 +233,12 @@ public:
 
   virtual void InclusionDirective(SourceLocation HashLoc,
                                   const Token &IncludeTok,
-                                  llvm::StringRef FileName,
+                                  StringRef FileName,
                                   bool IsAngled,
                                   const FileEntry *File,
                                   SourceLocation EndLoc,
-                                  llvm::StringRef SearchPath,
-                                  llvm::StringRef RelativePath) {
+                                  StringRef SearchPath,
+                                  StringRef RelativePath) {
     First->InclusionDirective(HashLoc, IncludeTok, FileName, IsAngled, File,
                               EndLoc, SearchPath, RelativePath);
     Second->InclusionDirective(HashLoc, IncludeTok, FileName, IsAngled, File,
@@ -246,32 +261,33 @@ public:
     Second->PragmaComment(Loc, Kind, Str);
   }
 
-  virtual void PragmaMessage(SourceLocation Loc, llvm::StringRef Str) {
+  virtual void PragmaMessage(SourceLocation Loc, StringRef Str) {
     First->PragmaMessage(Loc, Str);
     Second->PragmaMessage(Loc, Str);
   }
 
   virtual void PragmaDiagnosticPush(SourceLocation Loc,
-                                    llvm::StringRef Namespace) {
+                                    StringRef Namespace) {
     First->PragmaDiagnosticPush(Loc, Namespace);
     Second->PragmaDiagnosticPush(Loc, Namespace);
   }
 
   virtual void PragmaDiagnosticPop(SourceLocation Loc,
-                                    llvm::StringRef Namespace) {
+                                    StringRef Namespace) {
     First->PragmaDiagnosticPop(Loc, Namespace);
     Second->PragmaDiagnosticPop(Loc, Namespace);
   }
 
-  virtual void PragmaDiagnostic(SourceLocation Loc, llvm::StringRef Namespace,
-                                diag::Mapping mapping, llvm::StringRef Str) {
+  virtual void PragmaDiagnostic(SourceLocation Loc, StringRef Namespace,
+                                diag::Mapping mapping, StringRef Str) {
     First->PragmaDiagnostic(Loc, Namespace, mapping, Str);
     Second->PragmaDiagnostic(Loc, Namespace, mapping, Str);
   }
 
-  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI) {
-    First->MacroExpands(MacroNameTok, MI);
-    Second->MacroExpands(MacroNameTok, MI);
+  virtual void MacroExpands(const Token &MacroNameTok, const MacroInfo* MI,
+                            SourceRange Range) {
+    First->MacroExpands(MacroNameTok, MI, Range);
+    Second->MacroExpands(MacroNameTok, MI, Range);
   }
 
   virtual void MacroDefined(const Token &MacroNameTok, const MacroInfo *MI) {
@@ -284,6 +300,16 @@ public:
     Second->MacroUndefined(MacroNameTok, MI);
   }
 
+  virtual void Defined(const Token &MacroNameTok) {
+    First->Defined(MacroNameTok);
+    Second->Defined(MacroNameTok);
+  }
+
+  virtual void SourceRangeSkipped(SourceRange Range) {
+    First->SourceRangeSkipped(Range);
+    Second->SourceRangeSkipped(Range);
+  }
+
   /// If -- This hook is called whenever an #if is seen.
   virtual void If(SourceRange Range) {
     First->If(Range);
diff --git a/include/clang/Lex/PTHManager.h b/include/clang/Lex/PTHManager.h
index 094b7ef667981..25a49038a863d 100644
--- a/include/clang/Lex/PTHManager.h
+++ b/include/clang/Lex/PTHManager.h
@@ -30,7 +30,7 @@ namespace clang {
 
 class FileEntry;
 class PTHLexer;
-class Diagnostic;
+class DiagnosticsEngine;
 class FileSystemStatCache;
 
 class PTHManager : public IdentifierInfoLookup {
@@ -115,11 +115,11 @@ public:
   ///  Unlike the version in IdentifierTable, this returns a pointer instead
   ///  of a reference.  If the pointer is NULL then the IdentifierInfo cannot
   ///  be found.
-  IdentifierInfo *get(llvm::StringRef Name);
+  IdentifierInfo *get(StringRef Name);
 
   /// Create - This method creates PTHManager objects.  The 'file' argument
   ///  is the name of the PTH file.  This method returns NULL upon failure.
-  static PTHManager *Create(const std::string& file, Diagnostic &Diags);
+  static PTHManager *Create(const std::string& file, DiagnosticsEngine &Diags);
 
   void setPreprocessor(Preprocessor *pp) { PP = pp; }
 
diff --git a/include/clang/Lex/Pragma.h b/include/clang/Lex/Pragma.h
index c6ab35c19c1ea..4868811e7036f 100644
--- a/include/clang/Lex/Pragma.h
+++ b/include/clang/Lex/Pragma.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_PRAGMA_H
 #define LLVM_CLANG_PRAGMA_H
 
+#include "clang/Basic/LLVM.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include <cassert>
@@ -58,11 +59,11 @@ namespace clang {
 class PragmaHandler {
   std::string Name;
 public:
-  explicit PragmaHandler(llvm::StringRef name) : Name(name) {}
+  explicit PragmaHandler(StringRef name) : Name(name) {}
   PragmaHandler() {}
   virtual ~PragmaHandler();
 
-  llvm::StringRef getName() const { return Name; }
+  StringRef getName() const { return Name; }
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken) = 0;
 
@@ -91,14 +92,14 @@ class PragmaNamespace : public PragmaHandler {
   ///
   llvm::StringMap<PragmaHandler*> Handlers;
 public:
-  explicit PragmaNamespace(llvm::StringRef Name) : PragmaHandler(Name) {}
+  explicit PragmaNamespace(StringRef Name) : PragmaHandler(Name) {}
   virtual ~PragmaNamespace();
 
   /// FindHandler - Check to see if there is already a handler for the
   /// specified name.  If not, return the handler for the null name if it
   /// exists, otherwise return null.  If IgnoreNull is true (the default) then
   /// the null handler isn't returned on failure to match.
-  PragmaHandler *FindHandler(llvm::StringRef Name,
+  PragmaHandler *FindHandler(StringRef Name,
                              bool IgnoreNull = true) const;
 
   /// AddPragma - Add a pragma to this namespace.
diff --git a/include/clang/Lex/PreprocessingRecord.h b/include/clang/Lex/PreprocessingRecord.h
index b38303a2f40b6..53da19e6c2961 100644
--- a/include/clang/Lex/PreprocessingRecord.h
+++ b/include/clang/Lex/PreprocessingRecord.h
@@ -16,6 +16,7 @@
 
 #include "clang/Lex/PPCallbacks.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/IdentifierTable.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/Allocator.h"
 #include <vector>
@@ -43,14 +44,14 @@ namespace clang {
   public:
     /// \brief The kind of preprocessed entity an object describes.
     enum EntityKind {
+      /// \brief Indicates a problem trying to load the preprocessed entity.
+      InvalidKind,
+
       /// \brief A macro expansion.
       MacroExpansionKind,
       
-      /// \brief A preprocessing directive whose kind is not specified.
-      ///
-      /// This kind will be used for any preprocessing directive that does not
-      /// have a more specific kind within the \c DirectiveKind enumeration.
-      PreprocessingDirectiveKind,
+      /// \defgroup Preprocessing directives
+      /// @{
       
       /// \brief A macro definition.
       MacroDefinitionKind,
@@ -59,7 +60,9 @@ namespace clang {
       /// #import, or \c #include_next.
       InclusionDirectiveKind,
 
-      FirstPreprocessingDirective = PreprocessingDirectiveKind,
+      /// @}
+
+      FirstPreprocessingDirective = MacroDefinitionKind,
       LastPreprocessingDirective = InclusionDirectiveKind
     };
 
@@ -73,7 +76,9 @@ namespace clang {
   protected:
     PreprocessedEntity(EntityKind Kind, SourceRange Range)
       : Kind(Kind), Range(Range) { }
-    
+
+    friend class PreprocessingRecord;
+
   public:
     /// \brief Retrieve the kind of preprocessed entity stored in this object.
     EntityKind getKind() const { return Kind; }
@@ -81,7 +86,11 @@ namespace clang {
     /// \brief Retrieve the source range that covers this entire preprocessed 
     /// entity.
     SourceRange getSourceRange() const { return Range; }
-    
+
+    /// \brief Returns true if there was a problem loading the preprocessed
+    /// entity.
+    bool isInvalid() const { return Kind == InvalidKind; }
+
     // Implement isa/cast/dyncast/etc.
     static bool classof(const PreprocessedEntity *) { return true; }
 
@@ -110,34 +119,6 @@ namespace clang {
     void operator delete(void* data) throw();
   };
   
-  /// \brief Records the location of a macro expansion.
-  class MacroExpansion : public PreprocessedEntity {
-    /// \brief The name of the macro being expanded.
-    IdentifierInfo *Name;
-    
-    /// \brief The definition of this macro.
-    MacroDefinition *Definition;
-    
-  public:
-    MacroExpansion(IdentifierInfo *Name, SourceRange Range,
-                   MacroDefinition *Definition)
-      : PreprocessedEntity(MacroExpansionKind, Range), Name(Name),
-        Definition(Definition) { }
-    
-    /// \brief The name of the macro being expanded.
-    IdentifierInfo *getName() const { return Name; }
-    
-    /// \brief The definition of the macro being expanded.
-    MacroDefinition *getDefinition() const { return Definition; }
-
-    // Implement isa/cast/dyncast/etc.
-    static bool classof(const PreprocessedEntity *PE) {
-      return PE->getKind() == MacroExpansionKind;
-    }
-    static bool classof(const MacroExpansion *) { return true; }
-
-  };
-  
   /// \brief Records the presence of a preprocessor directive.
   class PreprocessingDirective : public PreprocessedEntity {
   public:
@@ -156,21 +137,16 @@ namespace clang {
   class MacroDefinition : public PreprocessingDirective {
     /// \brief The name of the macro being defined.
     const IdentifierInfo *Name;
-    
-    /// \brief The location of the macro name in the macro definition.
-    SourceLocation Location;
 
   public:
-    explicit MacroDefinition(const IdentifierInfo *Name, SourceLocation Location,
-                             SourceRange Range)
-      : PreprocessingDirective(MacroDefinitionKind, Range), Name(Name), 
-        Location(Location) { }
+    explicit MacroDefinition(const IdentifierInfo *Name, SourceRange Range)
+      : PreprocessingDirective(MacroDefinitionKind, Range), Name(Name) { }
     
     /// \brief Retrieve the name of the macro being defined.
     const IdentifierInfo *getName() const { return Name; }
     
     /// \brief Retrieve the location of the macro name in the definition.
-    SourceLocation getLocation() const { return Location; }
+    SourceLocation getLocation() const { return getSourceRange().getBegin(); }
     
     // Implement isa/cast/dyncast/etc.
     static bool classof(const PreprocessedEntity *PE) {
@@ -178,6 +154,44 @@ namespace clang {
     }
     static bool classof(const MacroDefinition *) { return true; }
   };
+  
+  /// \brief Records the location of a macro expansion.
+  class MacroExpansion : public PreprocessedEntity {
+    /// \brief The definition of this macro or the name of the macro if it is
+    /// a builtin macro.
+    llvm::PointerUnion<IdentifierInfo *, MacroDefinition *> NameOrDef; 
+
+  public:
+    MacroExpansion(IdentifierInfo *BuiltinName, SourceRange Range)
+      : PreprocessedEntity(MacroExpansionKind, Range),
+        NameOrDef(BuiltinName) { }
+
+    MacroExpansion(MacroDefinition *Definition, SourceRange Range)
+      : PreprocessedEntity(MacroExpansionKind, Range),
+        NameOrDef(Definition) { }
+
+    /// \brief True if it is a builtin macro.
+    bool isBuiltinMacro() const { return NameOrDef.is<IdentifierInfo *>(); }
+    
+    /// \brief The name of the macro being expanded.
+    const IdentifierInfo *getName() const {
+      if (MacroDefinition *Def = getDefinition())
+        return Def->getName();
+      return NameOrDef.get<IdentifierInfo*>();
+    }
+    
+    /// \brief The definition of the macro being expanded. May return null if
+    /// this is a builtin macro.
+    MacroDefinition *getDefinition() const {
+      return NameOrDef.dyn_cast<MacroDefinition *>();
+    }
+
+    // Implement isa/cast/dyncast/etc.
+    static bool classof(const PreprocessedEntity *PE) {
+      return PE->getKind() == MacroExpansionKind;
+    }
+    static bool classof(const MacroExpansion *) { return true; }
+  };
 
   /// \brief Record the location of an inclusion directive, such as an
   /// \c #include or \c #import statement.
@@ -199,7 +213,7 @@ namespace clang {
   private:
     /// \brief The name of the file that was included, as written in
     /// the source.
-    llvm::StringRef FileName;
+    StringRef FileName;
 
     /// \brief Whether the file name was in quotation marks; otherwise, it was
     /// in angle brackets.
@@ -215,14 +229,14 @@ namespace clang {
 
   public:
     InclusionDirective(PreprocessingRecord &PPRec,
-                       InclusionKind Kind, llvm::StringRef FileName, 
+                       InclusionKind Kind, StringRef FileName, 
                        bool InQuotes, const FileEntry *File, SourceRange Range);
     
     /// \brief Determine what kind of inclusion directive this is.
     InclusionKind getKind() const { return static_cast<InclusionKind>(Kind); }
     
     /// \brief Retrieve the included file name as it was written in the source.
-    llvm::StringRef getFileName() const { return FileName; }
+    StringRef getFileName() const { return FileName; }
     
     /// \brief Determine whether the included file name was written in quotes;
     /// otherwise, it was written in angle brackets.
@@ -245,19 +259,24 @@ namespace clang {
   public:
     virtual ~ExternalPreprocessingRecordSource();
     
-    /// \brief Read any preallocated preprocessed entities from the external
-    /// source.
-    virtual void ReadPreprocessedEntities() = 0;
-    
-    /// \brief Read the preprocessed entity at the given offset.
-    virtual PreprocessedEntity *
-    ReadPreprocessedEntityAtOffset(uint64_t Offset) = 0;
+    /// \brief Read a preallocated preprocessed entity from the external source.
+    ///
+    /// \returns null if an error occurred that prevented the preprocessed
+    /// entity from being loaded.
+    virtual PreprocessedEntity *ReadPreprocessedEntity(unsigned Index) = 0;
+
+    /// \brief Returns a pair of [Begin, End) indices of preallocated
+    /// preprocessed entities that \arg Range encompasses.
+    virtual std::pair<unsigned, unsigned>
+        findPreprocessedEntitiesInRange(SourceRange Range) = 0;
   };
   
   /// \brief A record of the steps taken while preprocessing a source file,
   /// including the various preprocessing directives processed, macros 
   /// expanded, etc.
   class PreprocessingRecord : public PPCallbacks {
+    SourceManager &SourceMgr;
+
     /// \brief Whether we should include nested macro expansions in
     /// the preprocessing record.
     bool IncludeNestedMacroExpansions;
@@ -269,24 +288,64 @@ namespace clang {
     /// were seen.
     std::vector<PreprocessedEntity *> PreprocessedEntities;
     
+    /// \brief The set of preprocessed entities in this record that have been
+    /// loaded from external sources.
+    ///
+    /// The entries in this vector are loaded lazily from the external source,
+    /// and are referenced by the iterator using negative indices.
+    std::vector<PreprocessedEntity *> LoadedPreprocessedEntities;
+
+    /// \brief Global (loaded or local) ID for a preprocessed entity.
+    /// Negative values are used to indicate preprocessed entities
+    /// loaded from the external source while non-negative values are used to
+    /// indicate preprocessed entities introduced by the current preprocessor.
+    /// If M is the number of loaded preprocessed entities, value -M
+    /// corresponds to element 0 in the loaded entities vector, position -M+1
+    /// corresponds to element 1 in the loaded entities vector, etc.
+    typedef int PPEntityID;
+
+    PPEntityID getPPEntityID(unsigned Index, bool isLoaded) const {
+      return isLoaded ? PPEntityID(Index) - LoadedPreprocessedEntities.size()
+                      : Index;
+    }
+
     /// \brief Mapping from MacroInfo structures to their definitions.
-    llvm::DenseMap<const MacroInfo *, MacroDefinition *> MacroDefinitions;
+    llvm::DenseMap<const MacroInfo *, PPEntityID> MacroDefinitions;
 
     /// \brief External source of preprocessed entities.
     ExternalPreprocessingRecordSource *ExternalSource;
+
+    /// \brief Retrieve the preprocessed entity at the given ID.
+    PreprocessedEntity *getPreprocessedEntity(PPEntityID PPID);
+
+    /// \brief Retrieve the loaded preprocessed entity at the given index.
+    PreprocessedEntity *getLoadedPreprocessedEntity(unsigned Index);
     
-    /// \brief The number of preallocated entities (that are known to the
-    /// external source).
-    unsigned NumPreallocatedEntities;
-    
-    /// \brief Whether we have already loaded all of the preallocated entities.
-    mutable bool LoadedPreallocatedEntities;
-    
-    void MaybeLoadPreallocatedEntities() const ;
+    /// \brief Determine the number of preprocessed entities that were
+    /// loaded (or can be loaded) from an external source.
+    unsigned getNumLoadedPreprocessedEntities() const {
+      return LoadedPreprocessedEntities.size();
+    }
+
+    /// \brief Returns a pair of [Begin, End) indices of local preprocessed
+    /// entities that \arg Range encompasses.
+    std::pair<unsigned, unsigned>
+      findLocalPreprocessedEntitiesInRange(SourceRange Range) const;
+    unsigned findBeginLocalPreprocessedEntity(SourceLocation Loc) const;
+    unsigned findEndLocalPreprocessedEntity(SourceLocation Loc) const;
+
+    /// \brief Allocate space for a new set of loaded preprocessed entities.
+    ///
+    /// \returns The index into the set of loaded preprocessed entities, which
+    /// corresponds to the first newly-allocated entity.
+    unsigned allocateLoadedEntities(unsigned NumEntities);
+
+    /// \brief Register a new macro definition.
+    void RegisterMacroDefinition(MacroInfo *Macro, PPEntityID PPID);
     
   public:
-    /// \brief Construct 
-    explicit PreprocessingRecord(bool IncludeNestedMacroExpansions);
+    /// \brief Construct a new preprocessing record.
+    PreprocessingRecord(SourceManager &SM, bool IncludeNestedMacroExpansions);
     
     /// \brief Allocate memory in the preprocessing record.
     void *Allocate(unsigned Size, unsigned Align = 8) {
@@ -295,64 +354,180 @@ namespace clang {
     
     /// \brief Deallocate memory in the preprocessing record.
     void Deallocate(void *Ptr) { }
-    
-    size_t getTotalMemory() const {
-      return BumpAlloc.getTotalMemory();
-    }
-    
+
+    size_t getTotalMemory() const;
+
+    SourceManager &getSourceManager() const { return SourceMgr; }
+
     // Iteration over the preprocessed entities.
-    typedef std::vector<PreprocessedEntity *>::iterator iterator;
-    typedef std::vector<PreprocessedEntity *>::const_iterator const_iterator;
-    iterator begin(bool OnlyLocalEntities = false);
-    iterator end(bool OnlyLocalEntities = false);
-    const_iterator begin(bool OnlyLocalEntities = false) const;
-    const_iterator end(bool OnlyLocalEntities = false) const;
+    class iterator {
+      PreprocessingRecord *Self;
+      
+      /// \brief Position within the preprocessed entity sequence.
+      ///
+      /// In a complete iteration, the Position field walks the range [-M, N),
+      /// where negative values are used to indicate preprocessed entities
+      /// loaded from the external source while non-negative values are used to
+      /// indicate preprocessed entities introduced by the current preprocessor.
+      /// However, to provide iteration in source order (for, e.g., chained
+      /// precompiled headers), dereferencing the iterator flips the negative
+      /// values (corresponding to loaded entities), so that position -M 
+      /// corresponds to element 0 in the loaded entities vector, position -M+1
+      /// corresponds to element 1 in the loaded entities vector, etc. This
+      /// gives us a reasonably efficient, source-order walk.
+      PPEntityID Position;
+      
+    public:
+      typedef PreprocessedEntity *value_type;
+      typedef value_type&         reference;
+      typedef value_type*         pointer;
+      typedef std::random_access_iterator_tag iterator_category;
+      typedef int                 difference_type;
+      
+      iterator() : Self(0), Position(0) { }
+      
+      iterator(PreprocessingRecord *Self, int Position) 
+        : Self(Self), Position(Position) { }
+      
+      value_type operator*() const {
+        return Self->getPreprocessedEntity(Position);
+      }
+      
+      value_type operator[](difference_type D) {
+        return *(*this + D);
+      }
+      
+      iterator &operator++() {
+        ++Position;
+        return *this;
+      }
+      
+      iterator operator++(int) {
+        iterator Prev(*this);
+        ++Position;
+        return Prev;
+      }
+
+      iterator &operator--() {
+        --Position;
+        return *this;
+      }
+      
+      iterator operator--(int) {
+        iterator Prev(*this);
+        --Position;
+        return Prev;
+      }
+
+      friend bool operator==(const iterator &X, const iterator &Y) {
+        return X.Position == Y.Position;
+      }
+
+      friend bool operator!=(const iterator &X, const iterator &Y) {
+        return X.Position != Y.Position;
+      }
+      
+      friend bool operator<(const iterator &X, const iterator &Y) {
+        return X.Position < Y.Position;
+      }
+
+      friend bool operator>(const iterator &X, const iterator &Y) {
+        return X.Position > Y.Position;
+      }
+
+      friend bool operator<=(const iterator &X, const iterator &Y) {
+        return X.Position < Y.Position;
+      }
+      
+      friend bool operator>=(const iterator &X, const iterator &Y) {
+        return X.Position > Y.Position;
+      }
+
+      friend iterator& operator+=(iterator &X, difference_type D) {
+        X.Position += D;
+        return X;
+      }
+
+      friend iterator& operator-=(iterator &X, difference_type D) {
+        X.Position -= D;
+        return X;
+      }
+
+      friend iterator operator+(iterator X, difference_type D) {
+        X.Position += D;
+        return X;
+      }
+
+      friend iterator operator+(difference_type D, iterator X) {
+        X.Position += D;
+        return X;
+      }
+
+      friend difference_type operator-(const iterator &X, const iterator &Y) {
+        return X.Position - Y.Position;
+      }
+
+      friend iterator operator-(iterator X, difference_type D) {
+        X.Position -= D;
+        return X;
+      }
+    };
+    friend class iterator;
+
+    /// \brief Begin iterator for all preprocessed entities.
+    iterator begin() {
+      return iterator(this, -(int)LoadedPreprocessedEntities.size());
+    }
+
+    /// \brief End iterator for all preprocessed entities.
+    iterator end() {
+      return iterator(this, PreprocessedEntities.size());
+    }
+
+    /// \brief Begin iterator for local, non-loaded, preprocessed entities.
+    iterator local_begin() {
+      return iterator(this, 0);
+    }
+
+    /// \brief End iterator for local, non-loaded, preprocessed entities.
+    iterator local_end() {
+      return iterator(this, PreprocessedEntities.size());
+    }
+
+    /// \brief Returns a pair of [Begin, End) iterators of preprocessed entities
+    /// that source range \arg R encompasses.
+    std::pair<iterator, iterator> getPreprocessedEntitiesInRange(SourceRange R);
 
     /// \brief Add a new preprocessed entity to this record.
     void addPreprocessedEntity(PreprocessedEntity *Entity);
     
     /// \brief Set the external source for preprocessed entities.
-    void SetExternalSource(ExternalPreprocessingRecordSource &Source,
-                           unsigned NumPreallocatedEntities);
+    void SetExternalSource(ExternalPreprocessingRecordSource &Source);
 
     /// \brief Retrieve the external source for preprocessed entities.
     ExternalPreprocessingRecordSource *getExternalSource() const {
       return ExternalSource;
     }
     
-    unsigned getNumPreallocatedEntities() const {
-      return NumPreallocatedEntities;
-    }
-    
-    /// \brief Set the preallocated entry at the given index to the given
-    /// preprocessed entity.
-    void SetPreallocatedEntity(unsigned Index, PreprocessedEntity *Entity);
-
-    /// \brief Register a new macro definition.
-    void RegisterMacroDefinition(MacroInfo *Macro, MacroDefinition *MD);
-                           
-    /// \brief Retrieve the preprocessed entity at the given index.
-    PreprocessedEntity *getPreprocessedEntity(unsigned Index) {
-      assert(Index < PreprocessedEntities.size() &&
-             "Out-of-bounds preprocessed entity");
-      return PreprocessedEntities[Index];
-    }
-    
     /// \brief Retrieve the macro definition that corresponds to the given
     /// \c MacroInfo.
     MacroDefinition *findMacroDefinition(const MacroInfo *MI);
-    
-    virtual void MacroExpands(const Token &Id, const MacroInfo* MI);
+        
+    virtual void MacroExpands(const Token &Id, const MacroInfo* MI,
+                              SourceRange Range);
     virtual void MacroDefined(const Token &Id, const MacroInfo *MI);
     virtual void MacroUndefined(const Token &Id, const MacroInfo *MI);
     virtual void InclusionDirective(SourceLocation HashLoc,
                                     const Token &IncludeTok,
-                                    llvm::StringRef FileName,
+                                    StringRef FileName,
                                     bool IsAngled,
                                     const FileEntry *File,
                                     SourceLocation EndLoc,
-                                    llvm::StringRef SearchPath,
-                                    llvm::StringRef RelativePath);
+                                    StringRef SearchPath,
+                                    StringRef RelativePath);
+
+    friend class ASTReader;
+    friend class ASTWriter;
   };
 } // end namespace clang
 
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index f6f3205099a0c..8b7743316f9cd 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -49,6 +49,7 @@ class PPCallbacks;
 class CodeCompletionHandler;
 class DirectoryLookup;
 class PreprocessingRecord;
+class ModuleLoader;
   
 /// Preprocessor - This object engages in a tight little dance with the lexer to
 /// efficiently preprocess tokens.  Lexers know only about tokens within a
@@ -56,17 +57,19 @@ class PreprocessingRecord;
 /// like the #include stack, token expansion, etc.
 ///
 class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
-  Diagnostic        *Diags;
-  LangOptions        Features;
-  const TargetInfo  &Target;
+  DiagnosticsEngine        *Diags;
+  LangOptions       &Features;
+  const TargetInfo  *Target;
   FileManager       &FileMgr;
   SourceManager     &SourceMgr;
   ScratchBuffer     *ScratchBuf;
   HeaderSearch      &HeaderInfo;
+  ModuleLoader      &TheModuleLoader;
 
   /// \brief External source of macros.
   ExternalPreprocessorSource *ExternalSource;
 
+  
   /// PTH - An optional PTHManager object used for getting tokens from
   ///  a token cache rather than lexing the original source file.
   llvm::OwningPtr<PTHManager> PTH;
@@ -90,6 +93,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   IdentifierInfo *Ident__has_attribute;            // __has_attribute
   IdentifierInfo *Ident__has_include;              // __has_include
   IdentifierInfo *Ident__has_include_next;         // __has_include_next
+  IdentifierInfo *Ident__has_warning;              // __has_warning
 
   SourceLocation DATELoc, TIMELoc;
   unsigned CounterValue;  // Next __COUNTER__ value.
@@ -102,7 +106,9 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   // State that is set before the preprocessor begins.
   bool KeepComments : 1;
   bool KeepMacroComments : 1;
-
+  bool SuppressIncludeNotFoundError : 1;
+  bool AutoModuleImport : 1;
+  
   // State that changes while the preprocessor runs:
   bool InMacroArgs : 1;            // True if parsing fn macro invocation args.
 
@@ -145,6 +151,29 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// \brief The file that we're performing code-completion for, if any.
   const FileEntry *CodeCompletionFile;
 
+  /// \brief The offset in file for the code-completion point.
+  unsigned CodeCompletionOffset;
+
+  /// \brief The location for the code-completion point. This gets instantiated
+  /// when the CodeCompletionFile gets #include'ed for preprocessing.
+  SourceLocation CodeCompletionLoc;
+
+  /// \brief The start location for the file of the code-completion point.
+  /// This gets instantiated when the CodeCompletionFile gets #include'ed
+  /// for preprocessing.
+  SourceLocation CodeCompletionFileLoc;
+
+  /// \brief The source location of the __import_module__ keyword we just
+  /// lexed, if any.
+  SourceLocation ModuleImportLoc;
+
+  /// \brief The source location of the currently-active
+  /// #pragma clang arc_cf_code_audited begin.
+  SourceLocation PragmaARCCFCodeAuditedLoc;
+  
+  /// \brief True if we hit the code-completion point.
+  bool CodeCompletionReached;
+
   /// \brief The number of bytes that we will initially skip when entering the
   /// main file, which is used when loading a precompiled preamble, along
   /// with a flag that indicates whether skipping this number of bytes will
@@ -165,7 +194,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   ///  if not expanding a macro.  This is an alias for either CurLexer or
   ///  CurPTHLexer.
   PreprocessorLexer *CurPPLexer;
-
+    
   /// CurLookup - The DirectoryLookup structure used to find the current
   /// FileEntry, if CurLexer is non-null and if applicable.  This allows us to
   /// implement #include_next and find directory-specific properties.
@@ -175,20 +204,31 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// expanding a macro.  One of CurLexer and CurTokenLexer must be null.
   llvm::OwningPtr<TokenLexer> CurTokenLexer;
 
+  /// \brief The kind of lexer we're currently working with.
+  enum CurLexerKind { 
+    CLK_Lexer, 
+    CLK_PTHLexer, 
+    CLK_TokenLexer, 
+    CLK_CachingLexer,
+    CLK_LexAfterModuleImport
+  } CurLexerKind;
+
   /// IncludeMacroStack - This keeps track of the stack of files currently
   /// #included, and macros currently being expanded from, not counting
   /// CurLexer/CurTokenLexer.
   struct IncludeStackInfo {
+    enum CurLexerKind     CurLexerKind;
     Lexer                 *TheLexer;
     PTHLexer              *ThePTHLexer;
     PreprocessorLexer     *ThePPLexer;
     TokenLexer            *TheTokenLexer;
     const DirectoryLookup *TheDirLookup;
 
-    IncludeStackInfo(Lexer *L, PTHLexer* P, PreprocessorLexer* PPL,
+    IncludeStackInfo(enum CurLexerKind K, Lexer *L, PTHLexer* P, 
+                     PreprocessorLexer* PPL,
                      TokenLexer* TL, const DirectoryLookup *D)
-      : TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL), TheTokenLexer(TL),
-        TheDirLookup(D) {}
+      : CurLexerKind(K), TheLexer(L), ThePTHLexer(P), ThePPLexer(PPL), 
+        TheTokenLexer(TL), TheDirLookup(D) {}
   };
   std::vector<IncludeStackInfo> IncludeMacroStack;
 
@@ -220,10 +260,6 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// previous macro value.
   llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
 
-  /// \brief Expansion source location for the last macro that expanded
-  /// to no tokens.
-  SourceLocation LastEmptyMacroExpansionLoc;
-
   // Various statistics we track for performance analysis.
   unsigned NumDirectives, NumIncluded, NumDefined, NumUndefined, NumPragma;
   unsigned NumIf, NumElse, NumEndif;
@@ -246,7 +282,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
   /// going to lex in the cache and when it finishes the tokens are removed
   /// from the end of the cache.
-  llvm::SmallVector<Token, 16> MacroExpandedTokens;
+  SmallVector<Token, 16> MacroExpandedTokens;
   std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
 
   /// \brief A record of the macro definitions and expansions that
@@ -257,7 +293,7 @@ class Preprocessor : public llvm::RefCountedBase<Preprocessor> {
   PreprocessingRecord *Record;
   
 private:  // Cached tokens state.
-  typedef llvm::SmallVector<Token, 1> CachedTokensTy;
+  typedef SmallVector<Token, 1> CachedTokensTy;
 
   /// CachedTokens - Cached tokens are stored here when we do backtracking or
   /// lookahead. They are "lexed" by the CachingLex() method.
@@ -291,19 +327,27 @@ private:  // Cached tokens state.
   MacroInfo *getInfoForMacro(IdentifierInfo *II) const;
   
 public:
-  Preprocessor(Diagnostic &diags, const LangOptions &opts,
-               const TargetInfo &target,
+  Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
+               const TargetInfo *target,
                SourceManager &SM, HeaderSearch &Headers,
+               ModuleLoader &TheModuleLoader,
                IdentifierInfoLookup *IILookup = 0,
-               bool OwnsHeaderSearch = false);
+               bool OwnsHeaderSearch = false,
+               bool DelayInitialization = false);
 
   ~Preprocessor();
 
-  Diagnostic &getDiagnostics() const { return *Diags; }
-  void setDiagnostics(Diagnostic &D) { Diags = &D; }
+  /// \brief Initialize the preprocessor, if the constructor did not already
+  /// perform the initialization.
+  ///
+  /// \param Target Information about the target.
+  void Initialize(const TargetInfo &Target);
+  
+  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
+  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
 
   const LangOptions &getLangOptions() const { return Features; }
-  const TargetInfo &getTargetInfo() const { return Target; }
+  const TargetInfo &getTargetInfo() const { return *Target; }
   FileManager &getFileManager() const { return FileMgr; }
   SourceManager &getSourceManager() const { return SourceMgr; }
   HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
@@ -325,6 +369,9 @@ public:
     return ExternalSource;
   }
 
+  /// \brief Retrieve the module loader associated with this preprocessor.
+  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
+  
   /// SetCommentRetentionState - Control whether or not the preprocessor retains
   /// comments in output.
   void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
@@ -334,6 +381,19 @@ public:
 
   bool getCommentRetentionState() const { return KeepComments; }
 
+  void SetSuppressIncludeNotFoundError(bool Suppress) {
+    SuppressIncludeNotFoundError = Suppress;
+  }
+
+  bool GetSuppressIncludeNotFoundError() {
+    return SuppressIncludeNotFoundError;
+  }
+
+  /// \brief Specify whether automatic module imports are enabled.
+  void setAutoModuleImport(bool AutoModuleImport = true) {
+    this->AutoModuleImport = AutoModuleImport;
+  }
+  
   /// isCurrentLexer - Return true if we are lexing directly from the specified
   /// lexer.
   bool isCurrentLexer(const PreprocessorLexer *L) const {
@@ -380,12 +440,6 @@ public:
   macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
   macro_iterator macro_end(bool IncludeExternalMacros = true) const;
 
-  /// \brief Expansion source location for the last macro that expanded
-  /// to no tokens.
-  SourceLocation getLastEmptyMacroExpansionLoc() const {
-    return LastEmptyMacroExpansionLoc;
-  }
-
   const std::string &getPredefines() const { return Predefines; }
   /// setPredefines - Set the predefines for this Preprocessor.  These
   /// predefines are automatically injected when parsing the main file.
@@ -397,25 +451,25 @@ public:
   /// pointers is preferred unless the identifier is already available as a
   /// string (this avoids allocation and copying of memory to construct an
   /// std::string).
-  IdentifierInfo *getIdentifierInfo(llvm::StringRef Name) const {
+  IdentifierInfo *getIdentifierInfo(StringRef Name) const {
     return &Identifiers.get(Name);
   }
 
   /// AddPragmaHandler - Add the specified pragma handler to the preprocessor.
   /// If 'Namespace' is non-null, then it is a token required to exist on the
   /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
-  void AddPragmaHandler(llvm::StringRef Namespace, PragmaHandler *Handler);
+  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
   void AddPragmaHandler(PragmaHandler *Handler) {
-    AddPragmaHandler(llvm::StringRef(), Handler);
+    AddPragmaHandler(StringRef(), Handler);
   }
 
   /// RemovePragmaHandler - Remove the specific pragma handler from
   /// the preprocessor. If \arg Namespace is non-null, then it should
   /// be the namespace that \arg Handler was added to. It is an error
   /// to remove a handler that has not been registered.
-  void RemovePragmaHandler(llvm::StringRef Namespace, PragmaHandler *Handler);
+  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
   void RemovePragmaHandler(PragmaHandler *Handler) {
-    RemovePragmaHandler(llvm::StringRef(), Handler);
+    RemovePragmaHandler(StringRef(), Handler);
   }
 
   /// \brief Add the specified comment handler to the preprocessor.
@@ -524,16 +578,17 @@ public:
   /// Lex - To lex a token from the preprocessor, just pull a token from the
   /// current lexer or macro object.
   void Lex(Token &Result) {
-    if (CurLexer)
-      CurLexer->Lex(Result);
-    else if (CurPTHLexer)
-      CurPTHLexer->Lex(Result);
-    else if (CurTokenLexer)
-      CurTokenLexer->Lex(Result);
-    else
-      CachingLex(Result);
+    switch (CurLexerKind) {
+    case CLK_Lexer: CurLexer->Lex(Result); break;
+    case CLK_PTHLexer: CurPTHLexer->Lex(Result); break;
+    case CLK_TokenLexer: CurTokenLexer->Lex(Result); break;
+    case CLK_CachingLexer: CachingLex(Result); break;
+    case CLK_LexAfterModuleImport: LexAfterModuleImport(Result); break;
+    }
   }
 
+  void LexAfterModuleImport(Token &Result);
+  
   /// LexNonComment - Lex a token.  If it's a comment, keep lexing until we get
   /// something not a comment.  This is useful in -E -C mode where comments
   /// would foul up preprocessor directive handling.
@@ -556,6 +611,14 @@ public:
     DisableMacroExpansion = OldVal;
   }
 
+  /// LexUnexpandedNonComment - Like LexNonComment, but this disables macro
+  /// expansion of identifier tokens.
+  void LexUnexpandedNonComment(Token &Result) {
+    do
+      LexUnexpandedToken(Result);
+    while (Result.getKind() == tok::comment);
+  }
+
   /// LookAhead - This peeks ahead N tokens and returns that token without
   /// consuming any tokens.  LookAhead(0) returns the next token that would be
   /// returned by Lex(), LookAhead(1) returns the token after it, etc.  This
@@ -635,13 +698,46 @@ public:
   bool SetCodeCompletionPoint(const FileEntry *File,
                               unsigned Line, unsigned Column);
 
-  /// \brief Determine if this source location refers into the file
-  /// for which we are performing code completion.
-  bool isCodeCompletionFile(SourceLocation FileLoc) const;
-
   /// \brief Determine if we are performing code completion.
   bool isCodeCompletionEnabled() const { return CodeCompletionFile != 0; }
 
+  /// \brief Returns the location of the code-completion point.
+  /// Returns an invalid location if code-completion is not enabled or the file
+  /// containing the code-completion point has not been lexed yet.
+  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
+
+  /// \brief Returns the start location of the file of code-completion point.
+  /// Returns an invalid location if code-completion is not enabled or the file
+  /// containing the code-completion point has not been lexed yet.
+  SourceLocation getCodeCompletionFileLoc() const {
+    return CodeCompletionFileLoc;
+  }
+
+  /// \brief Returns true if code-completion is enabled and we have hit the
+  /// code-completion point.
+  bool isCodeCompletionReached() const { return CodeCompletionReached; }
+
+  /// \brief Note that we hit the code-completion point.
+  void setCodeCompletionReached() {
+    assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
+    CodeCompletionReached = true;
+    // Silence any diagnostics that occur after we hit the code-completion.
+    getDiagnostics().setSuppressAllDiagnostics(true);
+  }
+
+  /// \brief The location of the currently-active #pragma clang
+  /// arc_cf_code_audited begin.  Returns an invalid location if there
+  /// is no such pragma active.
+  SourceLocation getPragmaARCCFCodeAuditedLoc() const {
+    return PragmaARCCFCodeAuditedLoc;
+  }
+
+  /// \brief Set the location of the currently-active #pragma clang
+  /// arc_cf_code_audited begin.  An invalid location ends the pragma.
+  void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc) {
+    PragmaARCCFCodeAuditedLoc = Loc;
+  }
+
   /// \brief Instruct the preprocessor to skip part of the main
   /// the main source file.
   ///
@@ -657,11 +753,11 @@ public:
   /// Diag - Forwarding function for diagnostics.  This emits a diagnostic at
   /// the specified Token's location, translating the token's start
   /// position in the current buffer into a SourcePosition object for rendering.
-  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) {
+  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
     return Diags->Report(Loc, DiagID);
   }
 
-  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) {
+  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
     return Diags->Report(Tok.getLocation(), DiagID);
   }
 
@@ -672,8 +768,8 @@ public:
   /// \param buffer A buffer which will be used only if the token requires
   ///   "cleaning", e.g. if it contains trigraphs or escaped newlines
   /// \param invalid If non-null, will be set \c true if an error occurs.
-  llvm::StringRef getSpelling(SourceLocation loc,
-                              llvm::SmallVectorImpl<char> &buffer,
+  StringRef getSpelling(SourceLocation loc,
+                              SmallVectorImpl<char> &buffer,
                               bool *invalid = 0) const {
     return Lexer::getSpelling(loc, buffer, SourceMgr, Features, invalid);
   }
@@ -707,8 +803,8 @@ public:
   /// getSpelling - This method is used to get the spelling of a token into a
   /// SmallVector. Note that the returned StringRef may not point to the
   /// supplied buffer if a copy can be avoided.
-  llvm::StringRef getSpelling(const Token &Tok,
-                              llvm::SmallVectorImpl<char> &Buffer, 
+  StringRef getSpelling(const Token &Tok,
+                              SmallVectorImpl<char> &Buffer, 
                               bool *Invalid = 0) const;
 
   /// getSpellingOfSingleCharacterNumericConstant - Tok is a numeric constant
@@ -731,8 +827,9 @@ public:
   /// CreateString - Plop the specified string into a scratch buffer and set the
   /// specified token's location and length to it.  If specified, the source
   /// location provides a location of the expansion point of the token.
-  void CreateString(const char *Buf, unsigned Len,
-                    Token &Tok, SourceLocation SourceLoc = SourceLocation());
+  void CreateString(const char *Buf, unsigned Len, Token &Tok,
+                    SourceLocation ExpansionLocStart = SourceLocation(),
+                    SourceLocation ExpansionLocEnd = SourceLocation());
 
   /// \brief Computes the source location just past the end of the
   /// token at this source location.
@@ -892,16 +989,17 @@ public:
   /// caller is expected to provide a buffer that is large enough to hold the
   /// spelling of the filename, but is also expected to handle the case when
   /// this method decides to use a different buffer.
-  bool GetIncludeFilenameSpelling(SourceLocation Loc,llvm::StringRef &Filename);
+  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
 
   /// LookupFile - Given a "foo" or <foo> reference, look up the indicated file,
   /// return null on failure.  isAngled indicates whether the file reference is
   /// for system #include's or not (i.e. using <> instead of "").
-  const FileEntry *LookupFile(llvm::StringRef Filename,
+  const FileEntry *LookupFile(StringRef Filename,
                               bool isAngled, const DirectoryLookup *FromDir,
                               const DirectoryLookup *&CurDir,
-                              llvm::SmallVectorImpl<char> *SearchPath,
-                              llvm::SmallVectorImpl<char> *RelativePath);
+                              SmallVectorImpl<char> *SearchPath,
+                              SmallVectorImpl<char> *RelativePath,
+                              StringRef *SuggestedModule);
 
   /// GetCurLookup - The DirectoryLookup structure used to find the current
   /// FileEntry, if CurLexer is non-null and if applicable.  This allows us to
@@ -932,7 +1030,8 @@ public:
 private:
 
   void PushIncludeMacroStack() {
-    IncludeMacroStack.push_back(IncludeStackInfo(CurLexer.take(),
+    IncludeMacroStack.push_back(IncludeStackInfo(CurLexerKind,
+                                                 CurLexer.take(),
                                                  CurPTHLexer.take(),
                                                  CurPPLexer,
                                                  CurTokenLexer.take(),
@@ -946,6 +1045,7 @@ private:
     CurPPLexer = IncludeMacroStack.back().ThePPLexer;
     CurTokenLexer.reset(IncludeMacroStack.back().TheTokenLexer);
     CurDirLookup  = IncludeMacroStack.back().TheDirLookup;
+    CurLexerKind = IncludeMacroStack.back().CurLexerKind;
     IncludeMacroStack.pop_back();
   }
 
@@ -976,7 +1076,8 @@ private:
   /// already seen one so a #else directive is a duplicate.  When this returns,
   /// the caller can lex the first valid token.
   void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
-                                    bool FoundNonSkipPortion, bool FoundElse);
+                                    bool FoundNonSkipPortion, bool FoundElse,
+                                    SourceLocation ElseLoc = SourceLocation());
 
   /// PTHSkipExcludedConditionalBlock - A fast PTH version of
   ///  SkipExcludedConditionalBlock.
@@ -1006,7 +1107,7 @@ private:
   /// going to lex in the cache and when it finishes the tokens are removed
   /// from the end of the cache.
   Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
-                                  llvm::ArrayRef<Token> tokens);
+                                  ArrayRef<Token> tokens);
   void removeCachedMacroExpandedTokensOfLastLexer();
   friend void TokenLexer::ExpandFunctionArguments();
 
@@ -1081,7 +1182,8 @@ private:
   void HandleDigitDirective(Token &Tok);
   void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
   void HandleIdentSCCSDirective(Token &Tok);
-
+  void HandleMacroExportDirective(Token &Tok);
+  
   // File inclusion.
   void HandleIncludeDirective(SourceLocation HashLoc,
                               Token &Tok,
diff --git a/include/clang/Lex/PreprocessorLexer.h b/include/clang/Lex/PreprocessorLexer.h
index 7bf041df974f3..e2e30bf878374 100644
--- a/include/clang/Lex/PreprocessorLexer.h
+++ b/include/clang/Lex/PreprocessorLexer.h
@@ -30,6 +30,9 @@ protected:
   /// The SourceManager FileID corresponding to the file being lexed.
   const FileID FID;
 
+  /// \brief Number of SLocEntries before lexing the file.
+  unsigned InitialNumSLocEntries;
+
   //===--------------------------------------------------------------------===//
   // Context-specific lexing flags set by the preprocessor.
   //===--------------------------------------------------------------------===//
@@ -61,18 +64,16 @@ protected:
 
   /// ConditionalStack - Information about the set of #if/#ifdef/#ifndef blocks
   /// we are currently in.
-  llvm::SmallVector<PPConditionalInfo, 4> ConditionalStack;
+  SmallVector<PPConditionalInfo, 4> ConditionalStack;
 
   PreprocessorLexer(const PreprocessorLexer&);          // DO NOT IMPLEMENT
   void operator=(const PreprocessorLexer&); // DO NOT IMPLEMENT
   friend class Preprocessor;
 
-  PreprocessorLexer(Preprocessor *pp, FileID fid)
-    : PP(pp), FID(fid), ParsingPreprocessorDirective(false),
-      ParsingFilename(false), LexingRawMode(false) {}
+  PreprocessorLexer(Preprocessor *pp, FileID fid);
 
   PreprocessorLexer()
-    : PP(0),
+    : PP(0), InitialNumSLocEntries(0),
       ParsingPreprocessorDirective(false),
       ParsingFilename(false),
       LexingRawMode(false) {}
@@ -151,13 +152,18 @@ public:
     return FID;
   }
 
+  /// \brief Number of SLocEntries before lexing the file.
+  unsigned getInitialNumSLocEntries() const {
+    return InitialNumSLocEntries;
+  }
+
   /// getFileEntry - Return the FileEntry corresponding to this FileID.  Like
   /// getFileID(), this only works for lexers with attached preprocessors.
   const FileEntry *getFileEntry() const;
 
   /// \brief Iterator that traverses the current stack of preprocessor
   /// conditional directives (#if/#ifdef/#ifndef).
-  typedef llvm::SmallVectorImpl<PPConditionalInfo>::const_iterator 
+  typedef SmallVectorImpl<PPConditionalInfo>::const_iterator 
     conditional_iterator;
 
   conditional_iterator conditional_begin() const { 
diff --git a/include/clang/Lex/Token.h b/include/clang/Lex/Token.h
index 9cf11d9a64c4d..e6dd1607e88bc 100644
--- a/include/clang/Lex/Token.h
+++ b/include/clang/Lex/Token.h
@@ -96,7 +96,10 @@ public:
   /// constant, string, etc.
   bool isLiteral() const {
     return is(tok::numeric_constant) || is(tok::char_constant) ||
-           is(tok::string_literal) || is(tok::wide_string_literal) ||
+           is(tok::wide_char_constant) || is(tok::utf16_char_constant) ||
+           is(tok::utf32_char_constant) || is(tok::string_literal) ||
+           is(tok::wide_string_literal) || is(tok::utf8_string_literal) ||
+           is(tok::utf16_string_literal) || is(tok::utf32_string_literal) ||
            is(tok::angle_string_literal);
   }
 
diff --git a/include/clang/Lex/TokenConcatenation.h b/include/clang/Lex/TokenConcatenation.h
index 094990a6e317a..551300f402c2e 100644
--- a/include/clang/Lex/TokenConcatenation.h
+++ b/include/clang/Lex/TokenConcatenation.h
@@ -63,12 +63,9 @@ namespace clang {
                      const Token &Tok) const;
 
   private:
-    /// StartsWithL - Return true if the spelling of this token starts with 'L'.
-    bool StartsWithL(const Token &Tok) const;
-
-    /// IsIdentifierL - Return true if the spelling of this token is literally
-    /// 'L'.
-    bool IsIdentifierL(const Token &Tok) const;
+    /// IsIdentifierStringPrefix - Return true if the spelling of the token
+    /// is literally 'L', 'u', 'U', or 'u8'.
+    bool IsIdentifierStringPrefix(const Token &Tok) const;
   };
   } // end clang namespace
 
diff --git a/include/clang/Lex/TokenLexer.h b/include/clang/Lex/TokenLexer.h
index 45ff8a03442e7..1330ad5f31436 100644
--- a/include/clang/Lex/TokenLexer.h
+++ b/include/clang/Lex/TokenLexer.h
@@ -71,8 +71,10 @@ class TokenLexer {
   /// "source location address space".
   unsigned MacroStartSLocOffset;
 
-  /// \brief FileID/offset of the start of the macro definition.
-  std::pair<FileID, unsigned> MacroDefStartInfo;
+  /// \brief Location of the macro definition.
+  SourceLocation MacroDefStart;
+  /// \brief Length of the macro definition.
+  unsigned MacroDefLength;
 
   /// Lexical information about the expansion point of the macro: the identifier
   /// that the macro expanded from had these properties.
@@ -169,7 +171,15 @@ private:
   /// \brief If \arg loc is a FileID and points inside the current macro
   /// definition, returns the appropriate source location pointing at the
   /// macro expansion source location entry.
-  SourceLocation getMacroExpansionLocation(SourceLocation loc) const;
+  SourceLocation getExpansionLocForMacroDefLoc(SourceLocation loc) const;
+
+  /// \brief Creates SLocEntries and updates the locations of macro argument
+  /// tokens to their new expanded locations.
+  ///
+  /// \param ArgIdSpellLoc the location of the macro argument id inside the
+  /// macro definition.
+  void updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
+                                  Token *begin_tokens, Token *end_tokens);
 };
 
 }  // end namespace clang